From 134dd47c7a78870960985f9aa17a642c2483ee3f Mon Sep 17 00:00:00 2001 From: Satya Ortiz-Gagne Date: Tue, 10 Sep 2024 18:07:38 -0400 Subject: [PATCH 1/8] Fix llama3 generation --- benchmarks/llm/configs/llama3_70B_full.yaml | 2 +- benchmarks/llm/prepare.py | 14 +++++++++++--- config/base.yaml | 7 +++++++ 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/benchmarks/llm/configs/llama3_70B_full.yaml b/benchmarks/llm/configs/llama3_70B_full.yaml index ae5cf2afb..5833044d9 100644 --- a/benchmarks/llm/configs/llama3_70B_full.yaml +++ b/benchmarks/llm/configs/llama3_70B_full.yaml @@ -36,7 +36,7 @@ checkpointer: _component_: torchtune.utils.FullModelHFCheckpointer checkpoint_dir: /tmp/Meta-Llama-3.1-70B-Instruct/ checkpoint_files: [ - model-00001-of-00030.safetensors, + model-00001-of-00030.safetensors, model-00002-of-00030.safetensors, model-00003-of-00030.safetensors, model-00004-of-00030.safetensors, diff --git a/benchmarks/llm/prepare.py b/benchmarks/llm/prepare.py index 5f74d2a4e..3ae749ed9 100755 --- a/benchmarks/llm/prepare.py +++ b/benchmarks/llm/prepare.py @@ -23,7 +23,6 @@ class Arguments: recipe: str config: str = None - no_pretrained: bool = False @dataclass @@ -100,12 +99,19 @@ def load_model(recipe, cfg): def generate_weights(args, config): + is_done:Path = args.output_dir / "generated" + if is_done.exists(): + print(f"{args.output_dir}/['*.safetensors'] or ['*consolidated.*.pth'] already generated") + return + if config.get("safetensors", False): params_path = args.output_dir / "config.json" model = LlamaForCausalLM(LlamaConfig(**json.loads(params_path.read_text()))) # Avoid saving this as part of the config. del model.config._name_or_path - model.config.torch_dtype = torch.float16 + # Even if model if loaded with a config.torch_dtype == bf16, model.dtype + # seams to be f32. Force model.dtype to be bf16 + model.to(model.config.torch_dtype) model.save_pretrained(str(args.output_dir), safe_serialization=True) else: @@ -138,6 +144,8 @@ def generate_weights(args, config): conn.send(True) p.join() + is_done.touch() + def main(): parser = ArgumentParser() @@ -154,7 +162,7 @@ def main(): # huggingface_format = config.get("safetensors", False) - pretrained = not args.no_pretrained + pretrained = not config.get("no_pretrained", False) if not pretrained: # if we will generate the weights do not download anyweights diff --git a/config/base.yaml b/config/base.yaml index 7bd21fc30..1ee4b182c 100644 --- a/config/base.yaml +++ b/config/base.yaml @@ -537,6 +537,7 @@ llm-lora-single: inherits: _llm plan: method: per_gpu + argv: "{milabench_code}/recipes/lora_finetune_single_device.py": true --config: "{milabench_code}/configs/llama3_8B_lora_single_device.yaml" @@ -549,6 +550,7 @@ llm-lora-single: repo_id="meta-llama/Meta-Llama-3.1-8B": true batch_size=8: true gradient_accumulation_steps=8: true + no_pretrained=True: true llm-lora-ddp-gpus: @@ -569,6 +571,7 @@ llm-lora-ddp-gpus: repo_id="meta-llama/Meta-Llama-3.1-8B": true batch_size=8: true gradient_accumulation_steps=8: true + no_pretrained=True: true llm-lora-ddp-nodes: @@ -592,6 +595,7 @@ llm-lora-ddp-nodes: repo_id="meta-llama/Meta-Llama-3.1-8B": true batch_size=8: true gradient_accumulation_steps=8: true + no_pretrained=True: true num_machines: 2 requires_capabilities: @@ -617,6 +621,7 @@ llm-lora-mp-gpus: repo_id="meta-llama/Meta-Llama-3.1-70B": true batch_size=8: true gradient_accumulation_steps=1: true + no_pretrained=True: true llm-full-mp-gpus: inherits: _llm @@ -637,6 +642,7 @@ llm-full-mp-gpus: safetensors=true: true batch_size=2: true gradient_accumulation_steps=1: true + no_pretrained=True: true llm-full-mp-nodes: tags: @@ -660,6 +666,7 @@ llm-full-mp-nodes: safetensors=true: true batch_size=2: true gradient_accumulation_steps=1: true + no_pretrained=True: true num_machines: 2 requires_capabilities: From 660f1d4bd392729e497f4183c4f25b19120d832d Mon Sep 17 00:00:00 2001 From: Satya Ortiz-Gagne Date: Tue, 3 Sep 2024 16:06:26 -0400 Subject: [PATCH 2/8] Add covalent cloud aws ec2 infra and report --push covalent is not compatible with milabench as it requires sqlalchemy<2.0.0 Update .github/workflows/cloud-ci.yml Apply suggestions from code review Update .github/workflows/cloud-ci.yml Add azure covalent cloud infra Add multi-node on cloud * VM on the cloud might not have enough space on all partitions. Add a workaround which should cover most cases * Use branch and commit name to versionize reports directories * Fix parsing error when temperature is not available in nvidia-smi outputs * export MILABENCH_* env vars to remote Add docs Fix cloud instance name conflict This would prevent the CI or multiple contributors to run tests with the same config Fix github push in CI * Copy ssh key to allow connections from master to workers * Use local ip for manager's ip such that workers can find it and connect to it --- .github/workflows/cloud-ci.yml | 204 +++++++ .../_templates/simple/requirements.cpu.txt | 46 ++ config/base.yaml | 10 +- config/cloud-multinodes-system.yaml | 40 ++ config/cloud-system.yaml | 40 ++ config/examples/cloud-multinodes-system.yaml | 37 ++ config/examples/cloud-system.yaml | 30 + config/examples/test.yaml | 24 + docs/dev-usage.rst | 13 + docs/usage.rst | 92 +++ milabench/__init__.py | 5 + milabench/cli/__init__.py | 5 + milabench/cli/cloud.py | 214 +++++++ milabench/cli/install.py | 2 - milabench/cli/report.py | 43 +- milabench/cli/run.py | 20 +- milabench/commands/__init__.py | 26 +- milabench/common.py | 136 ++++- milabench/config.py | 14 + milabench/log.py | 13 +- milabench/multi.py | 49 +- milabench/remote.py | 129 +++-- milabench/scripts/badges/__main__.py | 25 + milabench/scripts/badges/requirements.txt | 1 + milabench/scripts/covalent/__main__.py | 229 ++++++++ milabench/scripts/covalent/requirements.txt | 3 + milabench/scripts/utils.py | 44 ++ milabench/system.py | 28 +- poetry.lock | 526 +++++++++--------- 29 files changed, 1726 insertions(+), 322 deletions(-) create mode 100644 .github/workflows/cloud-ci.yml create mode 100644 benchmarks/_templates/simple/requirements.cpu.txt create mode 100644 config/cloud-multinodes-system.yaml create mode 100644 config/cloud-system.yaml create mode 100644 config/examples/cloud-multinodes-system.yaml create mode 100644 config/examples/cloud-system.yaml create mode 100644 config/examples/test.yaml create mode 100644 milabench/cli/cloud.py create mode 100644 milabench/scripts/badges/__main__.py create mode 100644 milabench/scripts/badges/requirements.txt create mode 100644 milabench/scripts/covalent/__main__.py create mode 100644 milabench/scripts/covalent/requirements.txt create mode 100644 milabench/scripts/utils.py diff --git a/.github/workflows/cloud-ci.yml b/.github/workflows/cloud-ci.yml new file mode 100644 index 000000000..6db1697c7 --- /dev/null +++ b/.github/workflows/cloud-ci.yml @@ -0,0 +1,204 @@ +name: cloud-tests + +on: + # Runs for pull requests + pull_request: + branches: + - master + +permissions: + id-token: write + contents: write + +jobs: + cloud-tests: + strategy: + fail-fast: true + max-parallel: 1 + matrix: + system: ["1n:1g", "1n:4g", "2n:4g"] + include: + - arch: cuda + exclude: "no-cuda" + # - arch: rocm + # exclude : "no-rocm" + + runs-on: ubuntu-latest + environment: cloud-ci + + # Cancel previous jobs if a new version was pushed + concurrency: + group: "${{ github.ref }}-${{ matrix.arch }}-${{ matrix.system }}" + cancel-in-progress: true + + defaults: + run: + shell: bash -el {0} + + env: + MILABENCH_CONFIG: "config/standard.yaml" + MILABENCH_SYSTEM: "config/cloud-multinodes-system.yaml" + MILABENCH_BASE: "../output" + MILABENCH_ARGS: "" + MILABENCH_DASH: "no" + MILABENCH_HF_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }} + ARM_TENANT_ID: "${{ secrets.ARM_TENANT_ID }}" + ARM_SUBSCRIPTION_ID: "${{ secrets.ARM_SUBSCRIPTION_ID }}" + AZURE_CORE_OUTPUT: none + _MULTI_GPUS: "diffusion-gpus,dinov2-giant-gpus,lightning-gpus,llava-gpus,resnet152-ddp-gpus,llm-full-mp-gpus,llm-lora-ddp-gpus,llm-lora-mp-gpus" + _MULTI_NODES: "multinode" + + steps: + - uses: actions/checkout@v3 + with: + token: ${{ github.token }} + + - uses: actions/setup-python@v2 + with: + python-version: '3.10' + + # Follow + # https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/guides/service_principal_client_secret + # to generate a clientId as well as a clientSecret + - name: Azure login + uses: azure/login@v2 + with: + creds: | + { + "clientId": "${{ secrets.ARM_CLIENT_ID }}", + "clientSecret": "${{ secrets.ARM_CLIENT_SECRET }}", + "subscriptionId": "${{ secrets.ARM_SUBSCRIPTION_ID }}", + "tenantId": "${{ secrets.ARM_TENANT_ID }}" + } + + - name: dependencies + run: | + python -m pip install -U pip + python -m pip install -U poetry + poetry lock --no-update + poetry install + + - name: setup cloud credentials + run: | + mkdir -p ~/.aws + mkdir -p ~/.ssh/covalent + echo "${{ secrets.COVALENT_EC2_EXECUTOR_KEYPAIR }}" >~/.ssh/covalent/covalent-ec2-executor-keypair.pem + echo "[default]" >~/.aws/credentials + echo "aws_access_key_id=${{ secrets.AWS_ACCESS_KEY_ID }}" >>~/.aws/credentials + echo "aws_secret_access_key=${{ secrets.AWS_SECRET_ACCESS_KEY }}" >>~/.aws/credentials + chmod -R a-rwx,u+rwX ~/.aws ~/.ssh + + - name: start covalent server + run: | + poetry run -- python3 -m milabench.scripts.covalent serve start --develop + + - name: setup cloud + run: | + nodes=$(echo "${{ matrix.system }}" | cut -d":" -f1) + gpus=$(echo "${{ matrix.system }}" | cut -d":" -f2) + case "$nodes" in + "1n") + MILABENCH_SYSTEM="config/cloud-system.yaml" + EXCLUDE="$EXCLUDE,$_MULTI_NODES" + ;; + "2n") + MILABENCH_SYSTEM="config/cloud-multinodes-system.yaml" + SELECT="$SELECT,$_MULTI_NODES" + EXCLUDE="$EXCLUDE,$_MULTI_GPUS" + ;; + *) + exit 1 + ;; + esac + case "$gpus" in + "1g") + RUN_ON="azure__a100" + EXCLUDE="$EXCLUDE,$_MULTI_GPUS,$_MULTI_NODES" + ;; + # "2g") + # RUN_ON="azure__a100_x2" + # SELECT="$SELECT,$_MULTI_GPUS" + # ;; + "4g") + RUN_ON="azure__a100_x4" + SELECT="$SELECT,$_MULTI_GPUS" + ;; + *) + exit 1 + ;; + esac + + if [[ -z "$(echo "$SELECT" | cut -d"," -f1)" ]] + then + SELECT="$(echo "$SELECT" | cut -d"," -f2-)" + fi + + if [[ -z "$(echo "$EXCLUDE" | cut -d"," -f1)" ]] + then + EXCLUDE="$(echo "$EXCLUDE" | cut -d"," -f2-)" + fi + + if [[ ! -z "$SELECT" ]] + then + SELECT="--select $SELECT" + fi + + if [[ ! -z "$EXCLUDE" ]] + then + EXCLUDE="--exclude $EXCLUDE" + fi + + echo "RUN_ON=$RUN_ON" >>$GITHUB_ENV + + poetry run milabench cloud \ + --setup \ + --run-on $RUN_ON \ + --system "$MILABENCH_SYSTEM" >$MILABENCH_SYSTEM.$RUN_ON + + echo "MILABENCH_SYSTEM=$MILABENCH_SYSTEM.$RUN_ON" >>$GITHUB_ENV + echo "SELECT=$SELECT" >>$GITHUB_ENV + echo "EXCLUDE=$EXCLUDE" >>$GITHUB_ENV + + - name: install benchmarks + run: | + poetry run milabench install --variant ${{ matrix.arch }} $SELECT $EXCLUDE + + - name: prepare benchmarks + run: | + poetry run milabench prepare $SELECT $EXCLUDE + + - name: run benchmarks + run: | + poetry run milabench run $SELECT $EXCLUDE + + - name: Summary + run: | + git config credential.${{ github.server_url }}.username ${{ github.actor }} + git config credential.helper '!f() { test "$1" = get && echo "password=$GITHUB_TOKEN"; }; f' + git config --global user.email "github@ci.com" + git config --global user.name "GitHub CI" + poetry run milabench report --push + env: + GITHUB_TOKEN: ${{ github.token }} + + - name: DEBUG state file + if: always() + run: | + cat /tmp/milabench/covalent_venv/lib/python*/site-packages/covalent_azure_plugin/infra/*.tfstate + + - name: teardown cloud + if: always() + run: | + if [[ -f "${MILABENCH_SYSTEM%.*}" ]] + then + export MILABENCH_SYSTEM=${MILABENCH_SYSTEM%.*} + fi + poetry run milabench cloud \ + --teardown \ + --run-on $RUN_ON \ + --all + + - name: DEBUG logs + if: always() + run: | + cat ~/.cache/covalent/covalent_ui.log diff --git a/benchmarks/_templates/simple/requirements.cpu.txt b/benchmarks/_templates/simple/requirements.cpu.txt new file mode 100644 index 000000000..e0058b822 --- /dev/null +++ b/benchmarks/_templates/simple/requirements.cpu.txt @@ -0,0 +1,46 @@ +# +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: +# +# pip-compile --output-file=benchmarks/_template/requirements.cpu.txt benchmarks/_template/requirements.in +# +antlr4-python3-runtime==4.9.3 + # via omegaconf +asttokens==2.4.1 + # via giving +codefind==0.1.3 + # via ptera +executing==1.2.0 + # via varname +giving==0.4.2 + # via + # ptera + # voir +markdown-it-py==3.0.0 + # via rich +mdurl==0.1.2 + # via markdown-it-py +omegaconf==2.3.0 + # via voir +ovld==0.3.2 + # via voir +ptera==1.4.1 + # via voir +pygments==2.17.2 + # via rich +pynvml==11.5.0 + # via voir +pyyaml==6.0.1 + # via omegaconf +reactivex==4.0.4 + # via giving +rich==13.7.0 + # via voir +six==1.16.0 + # via asttokens +typing-extensions==4.10.0 + # via reactivex +varname==0.10.0 + # via giving +voir==0.2.12 + # via -r benchmarks/_template/requirements.in diff --git a/config/base.yaml b/config/base.yaml index 7bd21fc30..14836bda8 100644 --- a/config/base.yaml +++ b/config/base.yaml @@ -10,6 +10,8 @@ _defaults: gpu_load_threshold: 0.5 gpu_mem_threshold: 0.5 + num_machines: 1 + _torchvision: inherits: _defaults definition: ../benchmarks/torchvision @@ -26,7 +28,6 @@ _torchvision: --loader: pytorch --data: "{milabench_data}/FakeImageNet" - _torchvision_ddp: inherits: _defaults definition: ../benchmarks/torchvision_ddp @@ -112,7 +113,6 @@ _timm: --dataset: "FakeImageNet" --workers: "auto({n_worker}, 8)" - _accelerate_opt: inherits: _defaults tags: @@ -149,7 +149,6 @@ _accelerate_opt: use_deepspeed: true num_machines: 1 - fp16: inherits: _flops @@ -389,7 +388,6 @@ brax: --num-minibatches: 32 --num-envs: 8192 - _diffusion: inherits: _defaults definition: ../benchmarks/diffusion @@ -532,7 +530,6 @@ _llm: definition: ../benchmarks/llm install_group: torch - llm-lora-single: inherits: _llm plan: @@ -550,7 +547,6 @@ llm-lora-single: batch_size=8: true gradient_accumulation_steps=8: true - llm-lora-ddp-gpus: inherits: _llm plan: @@ -570,7 +566,6 @@ llm-lora-ddp-gpus: batch_size=8: true gradient_accumulation_steps=8: true - llm-lora-ddp-nodes: tags: - multinode @@ -597,7 +592,6 @@ llm-lora-ddp-nodes: requires_capabilities: - "len(nodes) >= ${num_machines}" - llm-lora-mp-gpus: inherits: _llm plan: diff --git a/config/cloud-multinodes-system.yaml b/config/cloud-multinodes-system.yaml new file mode 100644 index 000000000..4f7fae391 --- /dev/null +++ b/config/cloud-multinodes-system.yaml @@ -0,0 +1,40 @@ +system: + # Nodes list + nodes: + # Alias used to reference the node + - name: manager + # Use 1.1.1.1 as an ip placeholder + ip: 1.1.1.1 + port: 5000 + # Use this node as the master node or not + main: true + # User to use in remote milabench operations + user: user + + - name: node1 + ip: 1.1.1.1 + main: false + user: username + + # Cloud instances profiles + cloud_profiles: + azure__a100: + username: ubuntu + size: Standard_NC24ads_A100_v4 + location: eastus2 + disk_size: 512 + azure__a100_x2: + username: ubuntu + size: Standard_NC48ads_A100_v4 + location: eastus2 + disk_size: 512 + azure__a100_x4: + username: ubuntu + size: Standard_NC96ads_A100_v4 + location: eastus2 + disk_size: 512 + azure__a10_x2: + username: ubuntu + size: Standard_NV72ads_A10_v5 + location: eastus2 + disk_size: 512 diff --git a/config/cloud-system.yaml b/config/cloud-system.yaml new file mode 100644 index 000000000..056ef3640 --- /dev/null +++ b/config/cloud-system.yaml @@ -0,0 +1,40 @@ +system: + # Nodes list + nodes: + # Alias used to reference the node + - name: manager + # Use 1.1.1.1 as an ip placeholder + ip: 1.1.1.1 + port: 5000 + # Use this node as the master node or not + main: true + # User to use in remote milabench operations + user: user + + # Cloud instances profiles + cloud_profiles: + azure__a100: + username: ubuntu + size: Standard_NC24ads_A100_v4 + location: eastus2 + disk_size: 512 + azure__a100_x2: + username: ubuntu + size: Standard_NC48ads_A100_v4 + location: eastus2 + disk_size: 512 + azure__a100_x4: + username: ubuntu + size: Standard_NC96ads_A100_v4 + location: eastus2 + disk_size: 512 + azure__a10: + username: ubuntu + size: Standard_NV36ads_A10_v5 + location: eastus2 + disk_size: 512 + azure__a10_x2: + username: ubuntu + size: Standard_NV72ads_A10_v5 + location: eastus2 + disk_size: 512 diff --git a/config/examples/cloud-multinodes-system.yaml b/config/examples/cloud-multinodes-system.yaml new file mode 100644 index 000000000..5066af5eb --- /dev/null +++ b/config/examples/cloud-multinodes-system.yaml @@ -0,0 +1,37 @@ +system: + # Nodes list + nodes: + # Alias used to reference the node + - name: manager + # Use 1.1.1.1 as an ip placeholder + ip: 1.1.1.1 + # Use this node as the master node or not + main: true + # User to use in remote milabench operations + user: user + + - name: node1 + ip: 1.1.1.1 + main: false + user: username + + # Cloud instances profiles + cloud_profiles: + # The cloud platform to use in the form of {PLATFORM} or + # {PLATFORM}__{PROFILE_NAME} + azure: + # covalent-azure-plugin args + username: ubuntu + size: Standard_B1s + location: eastus2 + azure__free: + username: ubuntu + size: Standard_B2ats_v2 + location: eastus2 + ec2: + # covalent-ec2-plugin args + username: ubuntu + instance_type: t2.micro + volume_size: 8 + region: us-east-2 + state_id: 71669879043a3864225aabb94f91a2d4 diff --git a/config/examples/cloud-system.yaml b/config/examples/cloud-system.yaml new file mode 100644 index 000000000..b3d1f70aa --- /dev/null +++ b/config/examples/cloud-system.yaml @@ -0,0 +1,30 @@ +system: + # Nodes list + nodes: + # Alias used to reference the node + - name: manager + # Use 1.1.1.1 as an ip placeholder + ip: 1.1.1.1 + # Use this node as the master node or not + main: true + # User to use in remote milabench operations + user: user + + # Cloud instances profiles + cloud_profiles: + # The cloud platform to use in the form of {PLATFORM}__{PROFILE_NAME} + azure: + # covalent-azure-plugin args + username: ubuntu + size: Standard_B1s + location: eastus2 + azure__free: + username: ubuntu + size: Standard_B2ats_v2 + location: eastus2 + ec2: + # covalent-ec2-plugin args + username: ubuntu + instance_type: t2.micro + volume_size: 8 + region: us-east-2 diff --git a/config/examples/test.yaml b/config/examples/test.yaml new file mode 100644 index 000000000..4f74ac33b --- /dev/null +++ b/config/examples/test.yaml @@ -0,0 +1,24 @@ +_defaults: + max_duration: 600 + voir: + options: + stop: 60 + interval: "1s" + +test: + inherits: _defaults + group: simple + install_group: test + definition: ../../benchmarks/_templates/simple + plan: + method: njobs + n: 1 + +testing: + inherits: _defaults + definition: ../../benchmarks/_templates/stdout + group: stdout + install_group: test + plan: + method: njobs + n: 1 diff --git a/docs/dev-usage.rst b/docs/dev-usage.rst index 42a9871e2..58d66fb0c 100644 --- a/docs/dev-usage.rst +++ b/docs/dev-usage.rst @@ -97,3 +97,16 @@ milabench compare ~~~~~~~~~~~~~~~~~ TODO. + +Using milabench on the cloud +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Milabench uses `Terraform `_ through +`Covalent `_. To add support for a new cloud +platform you will need to develop a new clovalent plugin with it's Terraform +config. An example is the +`covalent-azure-plugin `_. +The interesting parts would be: + +* `Terraform provider's related plugin arguments `_ +* `Terraform provider's configuration `_ diff --git a/docs/usage.rst b/docs/usage.rst index ecea88b75..b2a25d85d 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -69,3 +69,95 @@ The following command will print out a report of the tests that ran, the metrics milabench report --runs $MILABENCH_BASE/runs/some_specific_run --html report.html The report will also print out a score based on a weighting of the metrics, as defined in the file ``$MILABENCH_CONFIG`` points to. + + +Use milabench on the cloud +~~~~~~~~~~~~~~~~~~~~~~~~~~ + + +Setup Terraform and a free Azure account +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +1. | Install azure cli (it does not need to be in the same environment than + milabench) + | ``pip install azure-cli`` + +2. Setup a free account on + `azure.microsoft.com `_ + +3. Follow instructions in the + `azurerm documentation `_ + to generate a ``ARM_CLIENT_ID`` as well as a ``ARM_CLIENT_SECRET``. If you + don't have the permissions to create / assign a role to a service principal, + you can ignore the ``az ad sp create-for-rbac`` command to work directly with + your ``ARM_TENANT_ID`` and ``ARM_SUBSCRIPTION_ID`` + +4. `Install Terraform `_ + +5. Configure the ``azurerm`` Terraform provider by + `exporting the environment variables `_ + + +Create a cloud system configuration +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Add a ``cloud_profiles`` section to the ``system`` configuration which lists the +supported cloud profiles. + +.. notes:: + + Nodes that should be created on the cloud should have the ``1.1.1.1`` ip + address placeholder. Other ip addresses will be used as-is and no cloud + instance will be created for that node + +.. notes:: + + A cloud profile entry needs to start with a covalent plugin (e.g. `azure`). To + define multiple profiles on the same cloud platform, use the form + ``{PLATFORM}__{PROFILE_NAME}`` (e.g. ``azure__profile``). All cloud profile + attributes will be used as is as argument for the target covalent plugin + +.. code-block:: yaml + + system: + nodes: + - name: manager + # Use 1.1.1.1 as an ip placeholder + ip: 1.1.1.1 + main: true + user: + - name: node1 + ip: 1.1.1.1 + main: false + user: + + # Cloud instances profiles + cloud_profiles: + # The cloud platform to use in the form of {PLATFORM} or + # {PLATFORM}__{PROFILE_NAME} + azure__free: + # covalent-azure-plugin args + username: ubuntu + size: Standard_B2ats_v2 + location: eastus2 + # state_prefix and state_id can be set to force a specific cloud + # instance id + # state_prefix: cloud-ci + # state_id: 849897_bivunaku + + +Run milabench on the cloud +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +1. | Initialize the cloud instances + | ``milabench cloud --system {{SYSTEM_CONFIG.YAML}} --setup --run-on {{PROFILE}} >{{SYSTEM_CLOUD_CONFIG.YAML}}`` + +2. | Prepare, install and run milabench + | ``milabench [prepare|install|run] --system {{SYSTEM_CLOUD_CONFIG.YAML}}`` + +3. | Destroy the cloud instances + | ``milabench teardown --system {{SYSTEM_CLOUD_CONFIG.YAML}} --run-on {{PROFILE}}`` + | or + | ``milabench teardown --system {{SYSTEM_CLOUD_CONFIG.YAML}} --run-on {{PLATFORM}} --all`` + | to destroy not just a single cloud instance but all instances on a + specified platform that were instanced from the current local machine diff --git a/milabench/__init__.py b/milabench/__init__.py index e69de29bb..ac33e6bb3 100644 --- a/milabench/__init__.py +++ b/milabench/__init__.py @@ -0,0 +1,5 @@ +import pathlib + +ROOT_FOLDER = pathlib.Path(__file__).resolve().parent.parent +CONFIG_FOLDER = ROOT_FOLDER / "config" +BENCHMARK_FOLDER = ROOT_FOLDER / "benchmarks" diff --git a/milabench/cli/__init__.py b/milabench/cli/__init__.py index 5a1f122c5..34b6cf99a 100644 --- a/milabench/cli/__init__.py +++ b/milabench/cli/__init__.py @@ -3,6 +3,7 @@ from coleo import run_cli +from .cloud import cli_cloud from .compare import cli_compare from .dev import cli_dev from .install import cli_install @@ -45,6 +46,10 @@ def pin(): """Pin the benchmarks' dependencies.""" return cli_pin() + def cloud(): + """Setup cloud instances.""" + cli_cloud() + def dev(): """Create a shell in a benchmark's environment for development.""" return cli_dev() diff --git a/milabench/cli/cloud.py b/milabench/cli/cloud.py new file mode 100644 index 000000000..859cdab87 --- /dev/null +++ b/milabench/cli/cloud.py @@ -0,0 +1,214 @@ +from copy import deepcopy +import os +import subprocess +import sys +import warnings + +from coleo import Option, tooled +from omegaconf import OmegaConf +import yaml + +from milabench.fs import XPath +from milabench.utils import blabla + +from .. import ROOT_FOLDER +from ..common import get_multipack + +_SETUP = "setup" +_TEARDOWN = "teardown" +_LIST = "list" +_ACTIONS = (_SETUP, _TEARDOWN, _LIST) + + +def _flatten_cli_args(**kwargs): + return sum( + ( + (f"--{str(k).replace('_', '-')}", *([str(v)] if v is not None else [])) + for k, v in kwargs.items() + ), () + ) + + +def _or_sudo(cmd:str): + return f"( {cmd} || sudo {cmd} )" + + +def _get_common_dir(first_dir:XPath, second_dir:XPath): + f_parents, s_parents = ( + list(reversed((first_dir / "_").parents)), + list(reversed((second_dir / "_").parents)) + ) + f_parents, s_parents = ( + f_parents[:min(len(f_parents), len(s_parents))], + s_parents[:min(len(f_parents), len(s_parents))] + ) + while f_parents != s_parents: + f_parents = f_parents[:-1] + s_parents = s_parents[:-1] + if f_parents[-1] == XPath("/"): + # no common dir + return None + else: + return f_parents[-1] + + +def manage_cloud(pack, run_on, action="setup"): + assert run_on in pack.config["system"]["cloud_profiles"], f"{run_on} cloud profile not found in {list(pack.config['system']['cloud_profiles'].keys())}" + + key_map = { + "hostname":(lambda v: ("ip",v)), + "private_ip":(lambda v: ("internal_ip",v)), + "username":(lambda v: ("user",v)), + "ssh_key_file":(lambda v: ("key",v)), + # "env":(lambda v: ("env",[".", v, ";", "conda", "activate", "milabench", "&&"])), + } + plan_params = deepcopy(pack.config["system"]["cloud_profiles"][run_on]) + run_on, *profile = run_on.split("__") + profile = profile[0] if profile else "" + default_state_prefix = profile or run_on + default_state_id = "_".join((pack.config["hash"][:6], blabla())) + + local_base = pack.dirs.base.absolute() + local_data_dir = _get_common_dir(ROOT_FOLDER.parent, local_base.parent) + if local_data_dir is None: + local_data_dir = local_base.parent + remote_data_dir = XPath("/data") / local_data_dir.name + + nodes = iter(enumerate(pack.config["system"]["nodes"])) + for i, n in nodes: + if n["ip"] != "1.1.1.1": + continue + + plan_params["state_prefix"] = plan_params.get("state_prefix", default_state_prefix) + plan_params["state_id"] = plan_params.get("state_id", default_state_id) + plan_params["cluster_size"] = max(len(pack.config["system"]["nodes"]), i + 1) + plan_params["keep_alive"] = None + + import milabench.scripts.covalent as cv + + subprocess.run( + [ + sys.executable, + "-m", cv.__name__, + "serve", "start" + ] + , stdout=sys.stderr + , check=True + ) + + cmd = [ + sys.executable, + "-m", cv.__name__, + run_on, + f"--{action}", + *_flatten_cli_args(**plan_params) + ] + if action == _SETUP: + cmd += [ + "--", + "bash", "-c", + _or_sudo(f"mkdir -p '{local_data_dir.parent}'") + + " && " + _or_sudo(f"chmod a+rwX '{local_data_dir.parent}'") + + f" && mkdir -p '{remote_data_dir}'" + f" && ln -sfT '{remote_data_dir}' '{local_data_dir}'" + ] + p = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + stdout_chunks = [] + while True: + line = p.stdout.readline() + if not line: + break + line_str = line.decode("utf-8").strip() + stdout_chunks.append(line_str) + print(line_str, file=sys.stderr) + + if not line_str: + continue + try: + k, v = line_str.split("::>") + except ValueError: + continue + try: + k, v = key_map[k](v) + except KeyError: + warnings.warn(f"Ignoring invalid key received: {k}:{v}") + continue + if k == "ip" and n[k] != "1.1.1.1": + i, n = next(nodes) + n[k] = v + + _, stderr = p.communicate() + stderr = stderr.decode("utf-8").strip() + print(stderr, file=sys.stderr) + + if p.returncode != 0: + stdout = os.linesep.join(stdout_chunks) + raise subprocess.CalledProcessError( + p.returncode, + cmd, + stdout, + stderr + ) + + return pack.config["system"] + + +@tooled +def _setup(): + """Setup a cloud infrastructure""" + + # Setup cloud on target infra + run_on: Option & str + + mp = get_multipack() + setup_pack = mp.setup_pack() + system_config = manage_cloud(setup_pack, run_on, action=_SETUP) + del system_config["arch"] + + print(f"# hash::>{setup_pack.config['hash']}") + print(yaml.dump({"system": system_config})) + + +@tooled +def _teardown(): + """Teardown a cloud infrastructure""" + + # Teardown cloud instance on target infra + run_on: Option & str + + # Teardown all cloud instances + all: Option & bool = False + + overrides = {} + if all: + overrides = { + "*": OmegaConf.to_object(OmegaConf.from_dotlist([ + f"system.cloud_profiles.{run_on}.state_id='*'", + ])) + } + + mp = get_multipack(overrides=overrides) + setup_pack = mp.setup_pack() + manage_cloud(setup_pack, run_on, action=_TEARDOWN) + + +@tooled +def cli_cloud(): + """Manage cloud instances.""" + + # Setup a cloud infrastructure + setup: Option & bool = False + # Teardown a cloud infrastructure + teardown: Option & bool = False + + assert any((setup, teardown)) and not all((setup, teardown)) + + if setup: + _setup() + elif teardown: + _teardown() diff --git a/milabench/cli/install.py b/milabench/cli/install.py index 10d33a1da..6716926d8 100644 --- a/milabench/cli/install.py +++ b/milabench/cli/install.py @@ -53,8 +53,6 @@ def cli_install(args=None): if args.force: pack.dirs.venv.rm() - mp = get_multipack(run_name="install.{time}", overrides=overrides) - return run_with_loggers( mp.do_install(), loggers=[ diff --git a/milabench/cli/report.py b/milabench/cli/report.py index 9db3e5ca9..85fb6dfb0 100644 --- a/milabench/cli/report.py +++ b/milabench/cli/report.py @@ -1,10 +1,12 @@ +import glob import os import sys from dataclasses import dataclass, field from coleo import Option, config as configuration, tooled -from ..common import _error_report, _get_multipack, _read_reports +from ..common import Option, _error_report, _get_multipack, _push_reports, _read_reports +from ..fs import XPath from ..report import make_report from ..summary import make_summary @@ -12,12 +14,13 @@ # fmt: off @dataclass class Arguments: - runs: list = field(default_factory=list) + runs : list = field(default_factory=list) config : str = os.environ.get("MILABENCH_CONFIG", None) compare : str = None compare_gpus: bool = False html : str = None price : int = None + push : bool = False # fmt: on @@ -42,7 +45,10 @@ def arguments(): # Price per unit price: Option & int = None - return Arguments(runs, config, compare, compare_gpus, html, price) + # Push reports to repo + push: Option & bool = False + + return Arguments(runs, config, compare, compare_gpus, html, price, push) @tooled @@ -68,11 +74,6 @@ def cli_report(args=None): # ------ # 1 errors, details in HTML report. - reports = None - if args.runs: - reports = _read_reports(*args.runs) - summary = make_summary(reports) - if args.config: from milabench.common import arguments as multipack_args @@ -81,6 +82,25 @@ def cli_report(args=None): args.config = _get_multipack(margs, return_config=True) + assert args.config if args.push else None + + if not args.runs and args.config: + run_dirs = {XPath(pack_config["dirs"]["runs"]) for pack_config in args.config.values()} + filter = lambda _p: not any([XPath(_p).name.startswith(f"{prefix}.") for prefix in ("install", "prepare")]) + args.runs = sorted( + {_r + for _rd in run_dirs + for _r in glob.glob(str(_rd / "*.*.*/")) + if filter(_r) + }, + key=lambda _p: XPath(_p).name.split(".")[-2:] + ) + + reports = None + if args.runs: + reports = _read_reports(*args.runs) + summary = make_summary(reports) + make_report( summary, compare=args.compare, @@ -93,3 +113,10 @@ def cli_report(args=None): errdata=reports and _error_report(reports), stream=sys.stdout, ) + + if len(reports) and args.push: + reports_repo = next(iter( + XPath(pack_config["dirs"]["base"]) / "reports" + for pack_config in args.config.values() + )) + _push_reports(reports_repo, args.runs) diff --git a/milabench/cli/run.py b/milabench/cli/run.py index f5e75b702..b5e8e7f7c 100644 --- a/milabench/cli/run.py +++ b/milabench/cli/run.py @@ -3,6 +3,7 @@ from coleo import Option, tooled +from milabench.remote import is_remote from milabench.utils import validation_layers from ..common import ( @@ -63,14 +64,21 @@ def arguments(): return Arguments(run_name, repeat, fulltrace, report, dash, noterm, validations) - def _fetch_arch(mp): try: arch = next(iter(mp.packs.values())).config["system"]["arch"] except StopIteration: print("no selected bench") return None - + + +def _fetch_first_pack(mp): + try: + return next(iter(mp.packs.values())) + except StopIteration: + print("no selected bench") + return None + @tooled def cli_run(args=None): @@ -78,8 +86,6 @@ def cli_run(args=None): if args is None: args = arguments() - layers = validation_names(args.validations) - dash_class = { "short": ShortDashFormatter, "long": LongDashFormatter, @@ -87,8 +93,14 @@ def cli_run(args=None): }.get(args.dash, None) mp = get_multipack(run_name=args.run_name) + first_pack = _fetch_first_pack(mp) arch = _fetch_arch(mp) + layers = validation_names(args.validations) + if is_remote(first_pack): + # Remote execution will never send back rates + layers.remove("ensure_rate") + # Initialize the backend here so we can retrieve GPU stats init_arch(arch) diff --git a/milabench/commands/__init__.py b/milabench/commands/__init__.py index 3de44337a..ad281b23b 100644 --- a/milabench/commands/__init__.py +++ b/milabench/commands/__init__.py @@ -478,11 +478,10 @@ def _argv(self, **kwargs) -> List: host = f"{user}@{self.host}" if user else self.host argv = super()._argv(**kwargs) - argv.extend(["-oPasswordAuthentication=no"]) - argv.extend(["-p", str(self.port)]) - if key: - argv.append(f"-i{key}") + # scp apparently needs `-i` to be first + argv.insert(1, f"-i{key}") + argv.append(f"-p{self.port}") argv.append(host) return argv # + ["env", "-i"] @@ -495,21 +494,23 @@ def __init__( self, pack: pack.BasePackage, host: str, - directory: str, + src: str, *scp_argv, + dest: str = None, user: str = None, key: str = None, **kwargs, ) -> None: super().__init__(pack, host, "-r", *scp_argv, user=user, key=key, **kwargs) - self.dir = directory + self.src = src + self.dest = dest if dest is not None else self.src def _argv(self, **kwargs) -> List: argv = super()._argv(**kwargs) host = argv.pop() - argv.append(self.dir) - argv.append(f"{host}:{self.dir}") + argv.append(self.src) + argv.append(f"{host}:{self.dest}") return argv @@ -671,7 +672,9 @@ def make_base_executor(cls, executor, *args, **kwargs): main = nodes[0] # node[port] is for SSH - main_host = node_address(main) + # use internal ip if available such that workers can connect to the port + main_host = main.get("internal_ip", node_address(main)) + # add them as option so we could tweak them if necessary main_port = option("torchrun.port", int, default=29400) backend = option("torchrun.backend", str, default="c10d") @@ -948,6 +951,9 @@ def _get_main_and_workers(self): def _argv(self, **_) -> List: manager, nodes = self._get_main_and_workers() + # use internal ip if available such that workers can connect to the port + manager_ip = manager.get("internal_ip", node_address(manager)) + num_machines = max(1, len(nodes) + 1) # Cant do that maybe this run is constrained @@ -990,7 +996,7 @@ def _argv(self, **_) -> List: *deepspeed_argv, f"--gradient_accumulation_steps={self.pack.config.get('gradient_accumulation_steps', 1)}", f"--num_cpu_threads_per_process={cpu_per_process}", - f"--main_process_ip={manager['ip']}", + f"--main_process_ip={manager_ip}", f"--main_process_port={main_port}", f"--num_processes={nproc}", *self.accelerate_argv, diff --git a/milabench/common.py b/milabench/common.py index b533f892f..7cec181c4 100644 --- a/milabench/common.py +++ b/milabench/common.py @@ -1,16 +1,20 @@ +from copy import deepcopy import io import json import os import re import runpy +import subprocess import sys import traceback from dataclasses import dataclass, field from datetime import datetime from coleo import Option, default, tooled +import git from omegaconf import OmegaConf from voir.instruments.gpu import deduce_backend, select_backend +from milabench import ROOT_FOLDER from milabench.alt_async import proceed from milabench.utils import available_layers, blabla, multilogger @@ -210,6 +214,13 @@ def _get_multipack( if args.config is None: sys.exit("Error: CONFIG argument not provided and no $MILABENCH_CONFIG") + if args.system is None: + args.system = os.environ.get("MILABENCH_SYSTEM", None) + + if args.system is None: + if XPath(f"{args.config}.system").exists(): + args.system = f"{args.config}.system" + if args.select: args.select = set(args.select.split(",")) @@ -259,7 +270,7 @@ def _get_multipack( return selected_config else: return MultiPackage( - {name: get_pack(defn) for name, defn in selected_config.items()} + {name: get_pack(deepcopy(defn)) for name, defn in selected_config.items()} ) @@ -303,6 +314,129 @@ def _read_reports(*runs): return all_data +def _filter_reports(**reports): + _reports = {} + + for k, report in reports.items(): + config = next(iter(e for e in report if e["event"] == "config"), None) + if config is None: + continue + + if config["data"]["name"] != "remote": + _reports[k] = report + + return _reports + + +def _push_reports(reports_repo, runs): + _SVG_COLORS = { + "pass": "blue", + "partial": "yellow", + "failure": "red", + } + import milabench.scripts.badges as badges + + try: + reports_repo = git.Repo(str(reports_repo)) + except (git.exc.InvalidGitRepositoryError, git.exc.NoSuchPathError): + _repo = git.Repo(ROOT_FOLDER) + repo_url = next(iter(_r.url for _r in _repo.remotes if _r.name == "origin"), None) + reports_repo = git.Repo.clone_from(repo_url, str(reports_repo), branch="reports") + config_reader = _repo.config_reader() + config_writer = reports_repo.config_writer() + for section in config_reader.sections(): + if not section.startswith("credential"): + continue + for option in config_reader.options(section): + if not option.strip("_") == option: + continue + for value in config_reader.get_values(section, option): + config_writer.add_value(section, option, value) + config_writer.write() + + device_reports = {} + for run in runs: + reports = _read_reports(run) + reports = list(_filter_reports(**reports).values()) + + if not reports: + continue + + meta = [e["data"] for _r in reports for e in _r if e["event"] == "meta"] + + for gpu in (_ for _meta in meta for _ in _meta["accelerators"]["gpus"].values()): + device = gpu["product"].replace(" ", "_") + break + else: + for _meta in meta: + device = _meta["cpu"]["brand"].replace(" ", "_") + break + + build = meta[0]["milabench"]["tag"] + reports_dir = XPath(reports_repo.working_tree_dir) / build + + run = XPath(run) + try: + run.copy(reports_dir / device / run.name) + except FileExistsError: + pass + + for _f in (reports_dir / device / run.name).glob("*.stderr"): + _f.unlink() + + device_reports.setdefault((device, build), set()) + device_reports[(device, build)].update( + (reports_dir / device).glob("*/") + ) + + for (device, build), reports in device_reports.items(): + reports_dir = XPath(reports_repo.working_tree_dir) / build + reports = _read_reports(*reports) + reports = _filter_reports(**reports) + summary = make_summary(reports) + + successes = [s["successes"] for s in summary.values()] + failures = [s["failures"] for s in summary.values()] + + if sum(successes) == 0: + text = "failure" + elif any(failures): + text = "partial" + else: + text = "pass" + + result = subprocess.run( + [ + sys.executable, + "-m", badges.__name__, + "--left-text", device, + "--right-text", text, + "--right-color", _SVG_COLORS[text], + ], + capture_output=True, + check=True + ) + if result.returncode == 0: + (reports_dir / device / "badge.svg").write_text(result.stdout.decode("utf8")) + + with open(str(reports_dir / device / "README.md"), "wt") as _f: + _f.write("```\n") + make_report(summary, stream=_f) + _f.write("```\n") + + for cmd, _kwargs in ( + (["git", "pull"], {"check": True}), + (["git", "add", build], {"check": True}), + (["git", "commit", "-m", build], {"check": False}), + (["git", "push"], {"check": True}) + ): + subprocess.run( + cmd, + cwd=reports_repo.working_tree_dir, + **_kwargs + ) + + def _error_report(reports): out = {} for r, data in reports.items(): diff --git a/milabench/config.py b/milabench/config.py index ebc041060..4936054dc 100644 --- a/milabench/config.py +++ b/milabench/config.py @@ -1,4 +1,5 @@ import contextvars +import hashlib from copy import deepcopy import yaml @@ -72,6 +73,16 @@ def resolve_inheritance(bench_config, all_configs): return bench_config +def compute_config_hash(config): + config = deepcopy(config) + for entry in config: + config[entry]["dirs"] = {} + config[entry]["config_base"] = "" + config[entry]["config_file"] = "" + config[entry]["run_name"] = "" + return hashlib.md5(str(config).encode("utf8")).hexdigest() + + def finalize_config(name, bench_config): bench_config["name"] = name if "definition" in bench_config: @@ -139,6 +150,9 @@ def build_config(*config_files): for layer in _config_layers(config_files): all_configs = merge(all_configs, layer) + all_configs.setdefault("*", {}) + all_configs["*"]["hash"] = compute_config_hash(all_configs) + all_configs = build_matrix_bench(all_configs) for name, bench_config in all_configs.items(): diff --git a/milabench/log.py b/milabench/log.py index 167bab2f3..cda4faf92 100644 --- a/milabench/log.py +++ b/milabench/log.py @@ -423,6 +423,16 @@ def formatbyte(value): return f"{value // exp:4d} {name}" +_NO_DEFAULT_FLAG=("__NO_DEFAULT__",) +def _parse_int(value, default=_NO_DEFAULT_FLAG): + try: + return int(value) + except TypeError: + if default is not _NO_DEFAULT_FLAG: + return default + raise + + class LongDashFormatter(DashFormatter): def make_table(self): table = Table.grid(padding=(0, 3, 0, 0)) @@ -465,7 +475,8 @@ def on_data(self, entry, data, row): for gpuid, data in gpudata.items(): load = int(data.get("load", 0) * 100) currm, totalm = data.get("memory", [0, 0]) - temp = int(data.get("temperature", 0)) + # "temperature" is sometimes reported as None for some GPUs? A10? + temp = _parse_int(data.get("temperature", 0), 0) row[f"gpu:{gpuid}"] = ( f"{load:3d}% load | {currm:.0f}/{totalm:.0f} MB | {temp}C" ) diff --git a/milabench/multi.py b/milabench/multi.py index 7bb8490b1..05903768c 100644 --- a/milabench/multi.py +++ b/milabench/multi.py @@ -8,7 +8,7 @@ from voir.instruments.gpu import get_gpu_info from .capability import is_system_capable -from .commands import NJobs, PerGPU +from .commands import ListCommand, NJobs, PerGPU from .config import set_run_count from .fs import XPath from .config import get_base_folder @@ -17,6 +17,7 @@ is_main_local, is_multinode, is_remote, + milabench_remote_config, milabench_remote_install, milabench_remote_prepare, milabench_remote_run, @@ -90,13 +91,17 @@ async def copy_base_to_workers(setup): print("Coping main setup from this node to worker") # copy the main setup to the workers # so it copies the bench venv already, no need for python - from milabench.remote import copy_folder - from milabench.system import SystemConfig + from milabench.remote import INSTALL_FOLDER, copy_folder # we copy the entire content of base # FIXME: handle custom (venv, cache, data, etc...) directories # - copy_plan = copy_folder(setup, SystemConfig().base) + copy_plan = ListCommand( + *[ + copy_folder(setup, _dir) + for _dir in [str(INSTALL_FOLDER), str(setup.dirs.base)] + ] + ) remote_task = asyncio.create_task(copy_plan.execute()) await asyncio.wait([remote_task]) @@ -120,7 +125,10 @@ def setup_pack(self) -> Package: "dirs": pack.config["dirs"], "config_base": pack.config["config_base"], "config_file": pack.config["config_file"], + "plan": pack.config["plan"], "system": pack.config["system"], + "hash": pack.config["hash"], + "install_variant": pack.config["install_variant"], } ) @@ -158,6 +166,14 @@ async def do_install(self): if is_remote(setup): print("Current node is outside of our system") + + await asyncio.wait( + [ + asyncio.create_task(t.execute()) + for t in milabench_remote_config(setup, self.packs) + ] + ) + # We are outside system, setup the main node first remote_plan = milabench_remote_install(setup, setup_for="main") remote_task = asyncio.create_task(remote_plan.execute()) @@ -183,6 +199,13 @@ async def do_prepare(self): remote_task = None if is_remote(setup): + await asyncio.wait( + [ + asyncio.create_task(t.execute()) + for t in milabench_remote_config(setup, self.packs) + ] + ) + remote_plan = milabench_remote_prepare(setup, run_for="main") remote_task = asyncio.create_task(remote_plan.execute()) await asyncio.wait([remote_task]) @@ -204,6 +227,13 @@ async def do_run(self, repeat=1): setup = self.setup_pack() if is_remote(setup): + await asyncio.wait( + [ + asyncio.create_task(t.execute()) + for t in milabench_remote_config(setup, self.packs) + ] + ) + # if we are not on the main node right now # ssh to the main node and launch milabench remote_plan = milabench_remote_run(setup) @@ -222,6 +252,17 @@ async def do_run(self, repeat=1): exec_plan = make_execution_plan(pack, index, repeat) + # Generate and copy run phase config files (in particular + # voirconf-* from VoirCommand) + list(exec_plan.commands()) + await copy_base_to_workers( + setup.copy( + { + "tag": pack.config["tag"]+["nolog"], + } + ) + ) + print(repr(exec_plan)) await exec_plan.execute("run", timeout=True, timeout_delay=600) diff --git a/milabench/remote.py b/milabench/remote.py index 7e1eef85c..d7c71fe7d 100644 --- a/milabench/remote.py +++ b/milabench/remote.py @@ -1,6 +1,8 @@ +from copy import deepcopy import os import sys +from . import ROOT_FOLDER from .commands import ( CmdCommand, Command, @@ -10,7 +12,15 @@ VoidCommand, ) -INSTALL_FOLDER = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) +INSTALL_FOLDER = str(ROOT_FOLDER) + + +def milabench_env() -> list: + return [ + f"{envvar}={os.environ[envvar]}" + for envvar in os.environ + if envvar.split("_")[0] == "MILABENCH" and os.environ[envvar] + ] def scp(node, folder, dest=None) -> list: @@ -30,21 +40,40 @@ def scp(node, folder, dest=None) -> list: ] -def rsync(node, folder, dest=None) -> list: +def rsync(node, src=None, remote_src=None, dest=None, force=False) -> list: """Copy a folder from local node to remote node""" host = node["ip"] user = node["user"] + key = node.get("key", None) + key = f"-i{key}" if key else "" + + if isinstance(src, str): + src = [src] + + assert not src or not remote_src + assert src or remote_src if dest is None: - dest = os.path.abspath(os.path.join(folder, "..")) + _ = remote_src if remote_src else src[0] + dest = os.path.abspath(os.path.join(_, "..")) + + if remote_src: + remote_src = [f"{user}@{host}:{remote_src}"] + src = [] + else: + dest = f"{user}@{host}:{dest}" + remote_src = [] return [ "rsync", - "-av", + *(["--force", "--del"] if force else []), + "-aHv", "-e", - "ssh -oCheckHostIP=no -oStrictHostKeyChecking=no", - folder, - f"{user}@{host}:{dest}", + f"ssh {key} -oCheckHostIP=no -oStrictHostKeyChecking=no", + "--include=*/.git/*", + *[f"--exclude=*/{_dir}/*" for _dir in (".*", "tmp")], + *src, *remote_src, + dest, ] @@ -52,22 +81,11 @@ def pip_install_milabench(pack, node, folder) -> SSHCommand: host = node["ip"] user = node["user"] - cmd = ["pip", "install", "-e", folder] + cmd = ["python3", "-m", "pip", "install", "-Ue", folder] plan = CmdCommand(pack, *cmd) return SSHCommand(plan, host=host, user=user) -def milabench_remote_sync(pack, worker): - setup_for = "worker" - - # If we are outside the system prepare main only - # main will take care of preparing the workers - if is_remote(pack): - setup_for = "main" - - return milabench_remote_setup_plan(pack, setup_for) - - def should_run_for(worker, setup_for): if setup_for == "worker": return not worker.get("main", False) @@ -100,16 +118,18 @@ def worker_commands(pack, worker_plan, setup_for="worker"): def sshnode(node, cmd): host = node["ip"] user = node["user"] - port = node["sshport"] + port = node.get("sshport", 22) return SSHCommand(cmd, user=user, host=host, port=port) def copy_folder(pack, folder, setup_for="worker"): def copy_to_worker(nodepack, node): - return [ - sshnode(node, CmdCommand(nodepack, "mkdir", "-p", folder)), - CmdCommand(nodepack, *rsync(node, folder)) - ] + return SequenceCommand( + *[ + sshnode(node, CmdCommand(nodepack, "mkdir", "-p", folder)), + CmdCommand(nodepack, *rsync(node, folder, force=True)) + ] + ) return worker_commands(pack, copy_to_worker, setup_for=setup_for) @@ -123,16 +143,15 @@ def milabench_remote_setup_plan(pack, setup_for="worker") -> SequenceCommand: """ nodes = pack.config["system"]["nodes"] - copy = [] - node_packs = [] copy_source = copy_folder(pack, INSTALL_FOLDER, setup_for) install = [] - for i, node in enumerate(nodes): + for node in nodes: if should_run_for(node, setup_for): - install.append(pip_install_milabench(node_packs[i], node, INSTALL_FOLDER)) + node_pack = worker_pack(pack, node) + install.append(pip_install_milabench(node_pack, node, INSTALL_FOLDER)) return SequenceCommand( copy_source, @@ -140,6 +159,30 @@ def milabench_remote_setup_plan(pack, setup_for="worker") -> SequenceCommand: ) +def milabench_remote_fetch_reports_plan(pack, run_for="main") -> SequenceCommand: + """Copy milabench reports from remote + + Notes + ----- + Assume that the filesystem of remote node mirror local system. + """ + + nodes = pack.config["system"]["nodes"] + runs = pack.config["dirs"]["runs"] + + copy = [] + for node in nodes: + node_pack = None + + if should_run_for(node, run_for): + node_pack = worker_pack(pack, node) + copy.append(CmdCommand(node_pack, *rsync(node, remote_src=str(runs)))) + + return SequenceCommand( + ListCommand(*copy), + ) + + def worker_pack(pack, worker): if is_remote(pack): return pack.copy({}) @@ -164,7 +207,12 @@ def milabench_remote_command(pack, *command, run_for="worker") -> ListCommand: cmds.append( SSHCommand( - CmdCommand(worker_pack(pack, worker), "milabench", *command), + CmdCommand( + worker_pack(pack, worker), + "cd", f"{INSTALL_FOLDER}", "&&", + *milabench_env(), + "milabench", *command + ), host=host, user=user, key=key, @@ -208,6 +256,23 @@ def _sanity(pack, setup_for): assert is_remote(pack), "Only a remote node can setup the main node" +def milabench_remote_config(pack, packs): + for n in pack.config["system"]["nodes"]: + _cmds = [ + SSHCommand( + CmdCommand( + pack, + "(", "mkdir", "-p", str(ROOT_FOLDER.parent), pack.config["dirs"]["base"], ")", + "||", "(", "sudo", "mkdir", "-p", str(ROOT_FOLDER.parent), pack.config["dirs"]["base"], + "&&", "sudo", "chown", "-R", "$USER:$USER", str(ROOT_FOLDER.parent), pack.config["dirs"]["base"], ")", + ), + n["ip"], + ), + ] + + yield SequenceCommand(*_cmds) + + def milabench_remote_install(pack, setup_for="worker") -> SequenceCommand: """Copy milabench code, install milabench, execute milabench install""" _sanity(pack, setup_for) @@ -216,7 +281,6 @@ def milabench_remote_install(pack, setup_for="worker") -> SequenceCommand: return VoidCommand(pack) argv = sys.argv[2:] - return SequenceCommand( milabench_remote_setup_plan(pack, setup_for), milabench_remote_command(pack, "install", *argv, run_for=setup_for), @@ -243,4 +307,7 @@ def milabench_remote_run(pack) -> Command: return VoidCommand(pack) argv = sys.argv[2:] - return milabench_remote_command(pack, "run", *argv) + return SequenceCommand( + milabench_remote_command(pack, "run", *argv, "--run-name", pack.config["run_name"], run_for="main"), + milabench_remote_fetch_reports_plan(pack, run_for="main"), + ) diff --git a/milabench/scripts/badges/__main__.py b/milabench/scripts/badges/__main__.py new file mode 100644 index 000000000..e0a7bdc81 --- /dev/null +++ b/milabench/scripts/badges/__main__.py @@ -0,0 +1,25 @@ +import subprocess +import sys + + +def main(argv=None): + if argv is None: + argv = sys.argv[1:] + + try: + import pybadges as _ + except ImportError: + from ..utils import run_in_module_venv + check_if_module = "import pybadges" + return run_in_module_venv(__file__, check_if_module, argv) + + return subprocess.run([ + sys.executable, + "-m", + "pybadges", + *argv + ], check=True).returncode + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/milabench/scripts/badges/requirements.txt b/milabench/scripts/badges/requirements.txt new file mode 100644 index 000000000..2c1953bd5 --- /dev/null +++ b/milabench/scripts/badges/requirements.txt @@ -0,0 +1 @@ +pybadges diff --git a/milabench/scripts/covalent/__main__.py b/milabench/scripts/covalent/__main__.py new file mode 100644 index 000000000..d4de9d932 --- /dev/null +++ b/milabench/scripts/covalent/__main__.py @@ -0,0 +1,229 @@ +import argparse +import os +import pathlib +import subprocess +import sys +import tempfile + + +def serve(*argv): + return subprocess.run([ + "covalent", + *argv + ]).returncode + + +def _get_executor_kwargs(args): + return { + **{k:v for k,v in vars(args).items() if k not in ("setup", "teardown")}, + } + + +def executor(executor_cls, args, *argv): + import covalent as ct + + def _popen(cmd, *args, _env=None, **kwargs): + _env = _env if _env is not None else {} + + for envvar in _env.keys(): + envvar_val = _env[envvar] + + if not envvar_val: + continue + + envvar_val = pathlib.Path(envvar_val).expanduser() + if str(envvar_val) != _env[envvar]: + _env[envvar] = str(envvar_val) + + if "MILABENCH_CONFIG_CONTENT" in _env: + _config_dir = pathlib.Path(_env["MILABENCH_CONFIG"]).parent + with tempfile.NamedTemporaryFile("wt", dir=str(_config_dir), suffix=".yaml", delete=False) as _f: + _f.write(_env["MILABENCH_CONFIG_CONTENT"]) + _env["MILABENCH_CONFIG"] = _f.name + + try: + cmd = (str(pathlib.Path(cmd[0]).expanduser()), *cmd[1:]) + except IndexError: + pass + + cwd = kwargs.pop("cwd", None) + if cwd is not None: + cwd = str(pathlib.Path(cwd).expanduser()) + kwargs["cwd"] = cwd + + _env = {**os.environ.copy(), **kwargs.pop("env", {}), **_env} + + kwargs = { + **kwargs, + "env": _env, + "stdout": subprocess.PIPE, + "stderr": subprocess.PIPE, + } + p = subprocess.Popen(cmd, *args, **kwargs) + + stdout_chunks = [] + while True: + line = p.stdout.readline() + if not line: + break + line_str = line.decode("utf-8").strip() + stdout_chunks.append(line_str) + print(line_str) + + _, stderr = p.communicate() + stderr = stderr.decode("utf-8").strip() + stdout = os.linesep.join(stdout_chunks) + + if p.returncode != 0: + raise subprocess.CalledProcessError( + p.returncode, + (cmd, args, kwargs), + stdout, + stderr + ) + return p.returncode, stdout, stderr + + executor:ct.executor.BaseExecutor = executor_cls( + **_get_executor_kwargs(args), + ) + return_code = 0 + try: + if args.setup: + result = ct.dispatch_sync( + ct.lattice(executor.get_connection_attributes) + )().result + + assert result and result[0] + + all_connection_attributes, _ = result + master_host:str = None + for hostname, connection_attributes in all_connection_attributes.items(): + print(f"hostname::>{hostname}") + for attribute,value in connection_attributes.items(): + if attribute == "hostname": + continue + print(f"{attribute}::>{value}") + + master_host = master_host or hostname + + if len(all_connection_attributes) > 1: + # Add master node to known host to avoid unknown host error + # The authenticity of host '[hostname] ([IP address])' can't be established. + new_host = subprocess.run( + ["ssh-keyscan", master_host], + stdout=subprocess.PIPE, + check=True + ).stdout.decode("utf8") + known_hosts = pathlib.Path("~/.ssh/known_hosts").expanduser() + with known_hosts.open("at") as _f: + _f.write(new_host) + + # Add ssh file to master node to allow connections to worker + # nodes + ssh_key_file = all_connection_attributes[master_host]["ssh_key_file"] + fn = pathlib.Path(ssh_key_file) + result = ct.dispatch_sync( + ct.lattice(executor.cp_to_remote) + )(f".ssh/{fn.name.split('.')[0]}", str(fn)) + + assert result.status == ct.status.COMPLETED + + if argv: + result = ct.dispatch_sync( + ct.lattice(executor.list_running_instances) + )().result + + assert result + + dispatch_ids = set() + for connection_attributes in result.get( + (executor.state_prefix, executor.state_id), + {"env": None} + ).values(): + kwargs = { + **_get_executor_kwargs(args), + **connection_attributes + } + del kwargs["env"] + del kwargs["private_ip"] + + _executor:ct.executor.BaseExecutor = executor_cls(**kwargs) + + dispatch_ids.add( + ct.dispatch( + ct.lattice( + lambda:ct.electron(_popen, executor=_executor)(argv) + ), + disable_run=False + )() + ) + + for dispatch_id in dispatch_ids: + result = ct.get_result(dispatch_id=dispatch_id, wait=True).result + + _return_code, _, _ = result if result is not None else (1, "", "") + return_code = return_code or _return_code + + finally: + if args.teardown: + result = executor.stop_cloud_instance().result + assert result is not None + + return return_code + + +def main(argv=None): + if argv is None: + argv = sys.argv[1:] + + try: + import covalent as ct + except (KeyError, ImportError): + from ..utils import run_in_module_venv + check_if_module = "import covalent" + return run_in_module_venv(__file__, check_if_module, argv) + + parser = argparse.ArgumentParser() + subparsers = parser.add_subparsers() + subparser = subparsers.add_parser("serve") + subparser.add_argument(f"argv", nargs=argparse.REMAINDER) + for p in ("azure","ec2"): + try: + config = ct.get_config(f"executors.{p}") + except KeyError: + continue + subparser = subparsers.add_parser(p) + subparser.add_argument(f"--setup", action="store_true") + subparser.add_argument(f"--teardown", action="store_true") + for param, default in config.items(): + if param.startswith("_"): + continue + add_argument_kwargs = {} + if isinstance(default, bool): + add_argument_kwargs["action"] = "store_false" if default else "store_true" + else: + add_argument_kwargs["default"] = default + subparser.add_argument(f"--{param.replace('_', '-')}", **add_argument_kwargs) + + try: + cv_argv, argv = argv[:argv.index("--")], argv[argv.index("--")+1:] + except ValueError: + cv_argv, argv = argv, [] + + args = parser.parse_args(cv_argv) + + if cv_argv[0] == "serve": + assert not argv + return serve(*args.argv) + elif cv_argv[0] == "azure": + executor_cls = ct.executor.AzureExecutor + elif cv_argv[0] == "ec2": + executor_cls = ct.executor.EC2Executor + else: + raise + + return executor(executor_cls, args, *argv) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/milabench/scripts/covalent/requirements.txt b/milabench/scripts/covalent/requirements.txt new file mode 100644 index 000000000..b70efc793 --- /dev/null +++ b/milabench/scripts/covalent/requirements.txt @@ -0,0 +1,3 @@ +covalent==0.232 +covalent-ec2-plugin @ git+https://github.com/satyaog/covalent-ec2-plugin.git@feature/milabench +covalent-azure-plugin @ git+https://github.com/satyaog/covalent-azure-plugin.git@feature/milabench \ No newline at end of file diff --git a/milabench/scripts/utils.py b/milabench/scripts/utils.py new file mode 100644 index 000000000..5aec72d06 --- /dev/null +++ b/milabench/scripts/utils.py @@ -0,0 +1,44 @@ +import json +import pathlib +import subprocess +import sys + + +def get_venv(venv:pathlib.Path) -> dict: + activate = venv / "bin/activate" + if not activate.exists(): + raise FileNotFoundError(str(activate)) + env = subprocess.run( + f". '{activate}' && python3 -c 'import os ; import json ; print(json.dumps(dict(os.environ)))'", + shell=True, + capture_output=True + ).stdout + return json.loads(env) + + +def run_in_module_venv(module_main:str, check_if_module:str, argv:list=None): + module = pathlib.Path(module_main).resolve().parent + cache_dir = pathlib.Path(f"/tmp/milabench/{module.name}_venv") + python3 = str(cache_dir / "bin/python3") + try: + subprocess.run([python3, "-c", check_if_module], check=True, + stdout=sys.stderr) + except (FileNotFoundError, subprocess.CalledProcessError): + cache_dir.mkdir(parents=True, exist_ok=True) + subprocess.run([sys.executable, "-m", "virtualenv", str(cache_dir)], + check=True, stdout=sys.stderr) + subprocess.run([python3, "-m", "pip", "install", "-U", "pip"], + check=True, stdout=sys.stderr) + subprocess.run([ + python3, + "-m", + "pip", + "install", + "-r", + str(module / "requirements.txt") + ], stdout=sys.stderr, check=True) + subprocess.run([python3, "-c", check_if_module], check=True, stdout=sys.stderr) + return subprocess.call( + [python3, module_main, *argv], + env=get_venv(cache_dir) + ) \ No newline at end of file diff --git a/milabench/system.py b/milabench/system.py index d29f4cd27..74f8ab6d5 100644 --- a/milabench/system.py +++ b/milabench/system.py @@ -294,7 +294,11 @@ def _resolve_ip(ip): if not offline: # Resolve the IP try: - hostname, aliaslist, ipaddrlist = socket.gethostbyaddr(ip) + # Workaround error with `gethostbyaddr` on azure DNS (like + # `inmako.eastus2.cloudapp.azure.com`). A proper fix might be a + # correct network config in terraform. + # socket.herror: [Errno 1] Unknown host + hostname, aliaslist, ipaddrlist = socket.gethostbyname_ex(ip) lazy_raise = None except socket.herror as err: @@ -362,6 +366,10 @@ def _resolve_addresses(nodes): ("127.0.0.1" in ipaddrlist) or (hostname in ("localhost", socket.gethostname(), "127.0.0.1")) or (socket.gethostname().startswith(hostname)) + # Tmp workaround until networking on azure allows to associate the + # local hostname (`hostname.split(".")[0]`) with the public fqdn + # (hostname.split(".")[0].*.cloudapp.azure.com) + or (hostname.split(".")[0] == socket.gethostname()) or len(ip_list.intersection(ipaddrlist)) > 0 or any([is_loopback(ip) for ip in ipaddrlist]) ) @@ -399,13 +407,23 @@ def gethostname(host): def resolve_hostname(ip): try: - hostname, _, iplist = socket.gethostbyaddr(ip) + # Workaround error with `gethostbyaddr` on azure DNS (like + # `inmako.eastus2.cloudapp.azure.com`). A proper fix might be a + # correct network config in terraform. + # socket.herror: [Errno 1] Unknown host + hostname, _, iplist = socket.gethostbyname_ex(ip) for ip in iplist: if is_loopback(ip): return hostname, True - return hostname, hostname == socket.gethostname() + return hostname, ( + hostname == socket.gethostname() + # Tmp workaround until networking on azure allows to associate the + # local hostname (`hostname.split(".")[0]`) with the public fqdn + # (hostname.split(".")[0].*.cloudapp.azure.com) + or (hostname.split(".")[0] == socket.gethostname()) + ) except: if offline: @@ -460,9 +478,9 @@ def build_system_config(config_file, defaults=None, gpu=True): config = yaml.safe_load(cf) if defaults: - config = merge(defaults, config) + config["system"] = merge(defaults["system"], config["system"]) - system = config.get("system", {}) + system = config["system"] system_global.set(system) # capacity is only required if batch resizer is enabled diff --git a/poetry.lock b/poetry.lock index 1276e7e8f..ee426a145 100644 --- a/poetry.lock +++ b/poetry.lock @@ -76,32 +76,32 @@ files = [ [[package]] name = "attrs" -version = "23.2.0" +version = "24.2.0" description = "Classes Without Boilerplate" optional = false python-versions = ">=3.7" files = [ - {file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"}, - {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"}, + {file = "attrs-24.2.0-py3-none-any.whl", hash = "sha256:81921eb96de3191c8258c199618104dd27ac608d9366f5e35d011eae1867ede2"}, + {file = "attrs-24.2.0.tar.gz", hash = "sha256:5cfb1b9148b5b086569baec03f20d7b6bf3bcacc9a42bebf87ffaaca362f6346"}, ] [package.extras] -cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] -dev = ["attrs[tests]", "pre-commit"] -docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] -tests = ["attrs[tests-no-zope]", "zope-interface"] -tests-mypy = ["mypy (>=1.6)", "pytest-mypy-plugins"] -tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist[psutil]"] +benchmark = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-codspeed", "pytest-mypy-plugins", "pytest-xdist[psutil]"] +cov = ["cloudpickle", "coverage[toml] (>=5.3)", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] +dev = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pre-commit", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] +docs = ["cogapp", "furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier (<24.7)"] +tests = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] +tests-mypy = ["mypy (>=1.11.1)", "pytest-mypy-plugins"] [[package]] name = "babel" -version = "2.15.0" +version = "2.16.0" description = "Internationalization utilities" optional = false python-versions = ">=3.8" files = [ - {file = "Babel-2.15.0-py3-none-any.whl", hash = "sha256:08706bdad8d0a3413266ab61bd6c34d0c28d6e1e7badf40a2cebe67644e2e1fb"}, - {file = "babel-2.15.0.tar.gz", hash = "sha256:8daf0e265d05768bc6c7a314cf1321e9a123afc328cc635c18622a2f30a04413"}, + {file = "babel-2.16.0-py3-none-any.whl", hash = "sha256:368b5b98b37c06b7daf6696391c3240c938b37767d4584413e8438c5c435fa8b"}, + {file = "babel-2.16.0.tar.gz", hash = "sha256:d1f3554ca26605fe173f3de0c65f750f5a42f924499bf134de6423582298e316"}, ] [package.extras] @@ -125,33 +125,33 @@ url = "benchmate" [[package]] name = "black" -version = "24.4.2" +version = "24.8.0" description = "The uncompromising code formatter." optional = false python-versions = ">=3.8" files = [ - {file = "black-24.4.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:dd1b5a14e417189db4c7b64a6540f31730713d173f0b63e55fabd52d61d8fdce"}, - {file = "black-24.4.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e537d281831ad0e71007dcdcbe50a71470b978c453fa41ce77186bbe0ed6021"}, - {file = "black-24.4.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eaea3008c281f1038edb473c1aa8ed8143a5535ff18f978a318f10302b254063"}, - {file = "black-24.4.2-cp310-cp310-win_amd64.whl", hash = "sha256:7768a0dbf16a39aa5e9a3ded568bb545c8c2727396d063bbaf847df05b08cd96"}, - {file = "black-24.4.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:257d724c2c9b1660f353b36c802ccece186a30accc7742c176d29c146df6e474"}, - {file = "black-24.4.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:bdde6f877a18f24844e381d45e9947a49e97933573ac9d4345399be37621e26c"}, - {file = "black-24.4.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e151054aa00bad1f4e1f04919542885f89f5f7d086b8a59e5000e6c616896ffb"}, - {file = "black-24.4.2-cp311-cp311-win_amd64.whl", hash = "sha256:7e122b1c4fb252fd85df3ca93578732b4749d9be076593076ef4d07a0233c3e1"}, - {file = "black-24.4.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:accf49e151c8ed2c0cdc528691838afd217c50412534e876a19270fea1e28e2d"}, - {file = "black-24.4.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:88c57dc656038f1ab9f92b3eb5335ee9b021412feaa46330d5eba4e51fe49b04"}, - {file = "black-24.4.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:be8bef99eb46d5021bf053114442914baeb3649a89dc5f3a555c88737e5e98fc"}, - {file = "black-24.4.2-cp312-cp312-win_amd64.whl", hash = "sha256:415e686e87dbbe6f4cd5ef0fbf764af7b89f9057b97c908742b6008cc554b9c0"}, - {file = "black-24.4.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:bf10f7310db693bb62692609b397e8d67257c55f949abde4c67f9cc574492cc7"}, - {file = "black-24.4.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:98e123f1d5cfd42f886624d84464f7756f60ff6eab89ae845210631714f6db94"}, - {file = "black-24.4.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:48a85f2cb5e6799a9ef05347b476cce6c182d6c71ee36925a6c194d074336ef8"}, - {file = "black-24.4.2-cp38-cp38-win_amd64.whl", hash = "sha256:b1530ae42e9d6d5b670a34db49a94115a64596bc77710b1d05e9801e62ca0a7c"}, - {file = "black-24.4.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:37aae07b029fa0174d39daf02748b379399b909652a806e5708199bd93899da1"}, - {file = "black-24.4.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:da33a1a5e49c4122ccdfd56cd021ff1ebc4a1ec4e2d01594fef9b6f267a9e741"}, - {file = "black-24.4.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ef703f83fc32e131e9bcc0a5094cfe85599e7109f896fe8bc96cc402f3eb4b6e"}, - {file = "black-24.4.2-cp39-cp39-win_amd64.whl", hash = "sha256:b9176b9832e84308818a99a561e90aa479e73c523b3f77afd07913380ae2eab7"}, - {file = "black-24.4.2-py3-none-any.whl", hash = "sha256:d36ed1124bb81b32f8614555b34cc4259c3fbc7eec17870e8ff8ded335b58d8c"}, - {file = "black-24.4.2.tar.gz", hash = "sha256:c872b53057f000085da66a19c55d68f6f8ddcac2642392ad3a355878406fbd4d"}, + {file = "black-24.8.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:09cdeb74d494ec023ded657f7092ba518e8cf78fa8386155e4a03fdcc44679e6"}, + {file = "black-24.8.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:81c6742da39f33b08e791da38410f32e27d632260e599df7245cccee2064afeb"}, + {file = "black-24.8.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:707a1ca89221bc8a1a64fb5e15ef39cd755633daa672a9db7498d1c19de66a42"}, + {file = "black-24.8.0-cp310-cp310-win_amd64.whl", hash = "sha256:d6417535d99c37cee4091a2f24eb2b6d5ec42b144d50f1f2e436d9fe1916fe1a"}, + {file = "black-24.8.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:fb6e2c0b86bbd43dee042e48059c9ad7830abd5c94b0bc518c0eeec57c3eddc1"}, + {file = "black-24.8.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:837fd281f1908d0076844bc2b801ad2d369c78c45cf800cad7b61686051041af"}, + {file = "black-24.8.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:62e8730977f0b77998029da7971fa896ceefa2c4c4933fcd593fa599ecbf97a4"}, + {file = "black-24.8.0-cp311-cp311-win_amd64.whl", hash = "sha256:72901b4913cbac8972ad911dc4098d5753704d1f3c56e44ae8dce99eecb0e3af"}, + {file = "black-24.8.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:7c046c1d1eeb7aea9335da62472481d3bbf3fd986e093cffd35f4385c94ae368"}, + {file = "black-24.8.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:649f6d84ccbae73ab767e206772cc2d7a393a001070a4c814a546afd0d423aed"}, + {file = "black-24.8.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2b59b250fdba5f9a9cd9d0ece6e6d993d91ce877d121d161e4698af3eb9c1018"}, + {file = "black-24.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:6e55d30d44bed36593c3163b9bc63bf58b3b30e4611e4d88a0c3c239930ed5b2"}, + {file = "black-24.8.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:505289f17ceda596658ae81b61ebbe2d9b25aa78067035184ed0a9d855d18afd"}, + {file = "black-24.8.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:b19c9ad992c7883ad84c9b22aaa73562a16b819c1d8db7a1a1a49fb7ec13c7d2"}, + {file = "black-24.8.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1f13f7f386f86f8121d76599114bb8c17b69d962137fc70efe56137727c7047e"}, + {file = "black-24.8.0-cp38-cp38-win_amd64.whl", hash = "sha256:f490dbd59680d809ca31efdae20e634f3fae27fba3ce0ba3208333b713bc3920"}, + {file = "black-24.8.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:eab4dd44ce80dea27dc69db40dab62d4ca96112f87996bca68cd75639aeb2e4c"}, + {file = "black-24.8.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:3c4285573d4897a7610054af5a890bde7c65cb466040c5f0c8b732812d7f0e5e"}, + {file = "black-24.8.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9e84e33b37be070ba135176c123ae52a51f82306def9f7d063ee302ecab2cf47"}, + {file = "black-24.8.0-cp39-cp39-win_amd64.whl", hash = "sha256:73bbf84ed136e45d451a260c6b73ed674652f90a2b3211d6a35e78054563a9bb"}, + {file = "black-24.8.0-py3-none-any.whl", hash = "sha256:972085c618ee94f402da1af548a4f218c754ea7e5dc70acb168bfaca4c2542ed"}, + {file = "black-24.8.0.tar.gz", hash = "sha256:2500945420b6784c38b9ee885af039f5e7471ef284ab03fa35ecdde4688cd83f"}, ] [package.dependencies] @@ -393,63 +393,83 @@ development = ["black", "flake8", "mypy", "pytest", "types-colorama"] [[package]] name = "coverage" -version = "7.6.0" +version = "7.6.1" description = "Code coverage measurement for Python" optional = false python-versions = ">=3.8" files = [ - {file = "coverage-7.6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:dff044f661f59dace805eedb4a7404c573b6ff0cdba4a524141bc63d7be5c7fd"}, - {file = "coverage-7.6.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a8659fd33ee9e6ca03950cfdcdf271d645cf681609153f218826dd9805ab585c"}, - {file = "coverage-7.6.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7792f0ab20df8071d669d929c75c97fecfa6bcab82c10ee4adb91c7a54055463"}, - {file = "coverage-7.6.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d4b3cd1ca7cd73d229487fa5caca9e4bc1f0bca96526b922d61053ea751fe791"}, - {file = "coverage-7.6.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e7e128f85c0b419907d1f38e616c4f1e9f1d1b37a7949f44df9a73d5da5cd53c"}, - {file = "coverage-7.6.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:a94925102c89247530ae1dab7dc02c690942566f22e189cbd53579b0693c0783"}, - {file = "coverage-7.6.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:dcd070b5b585b50e6617e8972f3fbbee786afca71b1936ac06257f7e178f00f6"}, - {file = "coverage-7.6.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:d50a252b23b9b4dfeefc1f663c568a221092cbaded20a05a11665d0dbec9b8fb"}, - {file = "coverage-7.6.0-cp310-cp310-win32.whl", hash = "sha256:0e7b27d04131c46e6894f23a4ae186a6a2207209a05df5b6ad4caee6d54a222c"}, - {file = "coverage-7.6.0-cp310-cp310-win_amd64.whl", hash = "sha256:54dece71673b3187c86226c3ca793c5f891f9fc3d8aa183f2e3653da18566169"}, - {file = "coverage-7.6.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c7b525ab52ce18c57ae232ba6f7010297a87ced82a2383b1afd238849c1ff933"}, - {file = "coverage-7.6.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4bea27c4269234e06f621f3fac3925f56ff34bc14521484b8f66a580aacc2e7d"}, - {file = "coverage-7.6.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ed8d1d1821ba5fc88d4a4f45387b65de52382fa3ef1f0115a4f7a20cdfab0e94"}, - {file = "coverage-7.6.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:01c322ef2bbe15057bc4bf132b525b7e3f7206f071799eb8aa6ad1940bcf5fb1"}, - {file = "coverage-7.6.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:03cafe82c1b32b770a29fd6de923625ccac3185a54a5e66606da26d105f37dac"}, - {file = "coverage-7.6.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:0d1b923fc4a40c5832be4f35a5dab0e5ff89cddf83bb4174499e02ea089daf57"}, - {file = "coverage-7.6.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:4b03741e70fb811d1a9a1d75355cf391f274ed85847f4b78e35459899f57af4d"}, - {file = "coverage-7.6.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:a73d18625f6a8a1cbb11eadc1d03929f9510f4131879288e3f7922097a429f63"}, - {file = "coverage-7.6.0-cp311-cp311-win32.whl", hash = "sha256:65fa405b837060db569a61ec368b74688f429b32fa47a8929a7a2f9b47183713"}, - {file = "coverage-7.6.0-cp311-cp311-win_amd64.whl", hash = "sha256:6379688fb4cfa921ae349c76eb1a9ab26b65f32b03d46bb0eed841fd4cb6afb1"}, - {file = "coverage-7.6.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:f7db0b6ae1f96ae41afe626095149ecd1b212b424626175a6633c2999eaad45b"}, - {file = "coverage-7.6.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:bbdf9a72403110a3bdae77948b8011f644571311c2fb35ee15f0f10a8fc082e8"}, - {file = "coverage-7.6.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9cc44bf0315268e253bf563f3560e6c004efe38f76db03a1558274a6e04bf5d5"}, - {file = "coverage-7.6.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:da8549d17489cd52f85a9829d0e1d91059359b3c54a26f28bec2c5d369524807"}, - {file = "coverage-7.6.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0086cd4fc71b7d485ac93ca4239c8f75732c2ae3ba83f6be1c9be59d9e2c6382"}, - {file = "coverage-7.6.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1fad32ee9b27350687035cb5fdf9145bc9cf0a094a9577d43e909948ebcfa27b"}, - {file = "coverage-7.6.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:044a0985a4f25b335882b0966625270a8d9db3d3409ddc49a4eb00b0ef5e8cee"}, - {file = "coverage-7.6.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:76d5f82213aa78098b9b964ea89de4617e70e0d43e97900c2778a50856dac605"}, - {file = "coverage-7.6.0-cp312-cp312-win32.whl", hash = "sha256:3c59105f8d58ce500f348c5b56163a4113a440dad6daa2294b5052a10db866da"}, - {file = "coverage-7.6.0-cp312-cp312-win_amd64.whl", hash = "sha256:ca5d79cfdae420a1d52bf177de4bc2289c321d6c961ae321503b2ca59c17ae67"}, - {file = "coverage-7.6.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d39bd10f0ae453554798b125d2f39884290c480f56e8a02ba7a6ed552005243b"}, - {file = "coverage-7.6.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:beb08e8508e53a568811016e59f3234d29c2583f6b6e28572f0954a6b4f7e03d"}, - {file = "coverage-7.6.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b2e16f4cd2bc4d88ba30ca2d3bbf2f21f00f382cf4e1ce3b1ddc96c634bc48ca"}, - {file = "coverage-7.6.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6616d1c9bf1e3faea78711ee42a8b972367d82ceae233ec0ac61cc7fec09fa6b"}, - {file = "coverage-7.6.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ad4567d6c334c46046d1c4c20024de2a1c3abc626817ae21ae3da600f5779b44"}, - {file = "coverage-7.6.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:d17c6a415d68cfe1091d3296ba5749d3d8696e42c37fca5d4860c5bf7b729f03"}, - {file = "coverage-7.6.0-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:9146579352d7b5f6412735d0f203bbd8d00113a680b66565e205bc605ef81bc6"}, - {file = "coverage-7.6.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:cdab02a0a941af190df8782aafc591ef3ad08824f97850b015c8c6a8b3877b0b"}, - {file = "coverage-7.6.0-cp38-cp38-win32.whl", hash = "sha256:df423f351b162a702c053d5dddc0fc0ef9a9e27ea3f449781ace5f906b664428"}, - {file = "coverage-7.6.0-cp38-cp38-win_amd64.whl", hash = "sha256:f2501d60d7497fd55e391f423f965bbe9e650e9ffc3c627d5f0ac516026000b8"}, - {file = "coverage-7.6.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:7221f9ac9dad9492cecab6f676b3eaf9185141539d5c9689d13fd6b0d7de840c"}, - {file = "coverage-7.6.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ddaaa91bfc4477d2871442bbf30a125e8fe6b05da8a0015507bfbf4718228ab2"}, - {file = "coverage-7.6.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c4cbe651f3904e28f3a55d6f371203049034b4ddbce65a54527a3f189ca3b390"}, - {file = "coverage-7.6.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:831b476d79408ab6ccfadaaf199906c833f02fdb32c9ab907b1d4aa0713cfa3b"}, - {file = "coverage-7.6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:46c3d091059ad0b9c59d1034de74a7f36dcfa7f6d3bde782c49deb42438f2450"}, - {file = "coverage-7.6.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:4d5fae0a22dc86259dee66f2cc6c1d3e490c4a1214d7daa2a93d07491c5c04b6"}, - {file = "coverage-7.6.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:07ed352205574aad067482e53dd606926afebcb5590653121063fbf4e2175166"}, - {file = "coverage-7.6.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:49c76cdfa13015c4560702574bad67f0e15ca5a2872c6a125f6327ead2b731dd"}, - {file = "coverage-7.6.0-cp39-cp39-win32.whl", hash = "sha256:482855914928c8175735a2a59c8dc5806cf7d8f032e4820d52e845d1f731dca2"}, - {file = "coverage-7.6.0-cp39-cp39-win_amd64.whl", hash = "sha256:543ef9179bc55edfd895154a51792b01c017c87af0ebaae092720152e19e42ca"}, - {file = "coverage-7.6.0-pp38.pp39.pp310-none-any.whl", hash = "sha256:6fe885135c8a479d3e37a7aae61cbd3a0fb2deccb4dda3c25f92a49189f766d6"}, - {file = "coverage-7.6.0.tar.gz", hash = "sha256:289cc803fa1dc901f84701ac10c9ee873619320f2f9aff38794db4a4a0268d51"}, + {file = "coverage-7.6.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b06079abebbc0e89e6163b8e8f0e16270124c154dc6e4a47b413dd538859af16"}, + {file = "coverage-7.6.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:cf4b19715bccd7ee27b6b120e7e9dd56037b9c0681dcc1adc9ba9db3d417fa36"}, + {file = "coverage-7.6.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e61c0abb4c85b095a784ef23fdd4aede7a2628478e7baba7c5e3deba61070a02"}, + {file = "coverage-7.6.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fd21f6ae3f08b41004dfb433fa895d858f3f5979e7762d052b12aef444e29afc"}, + {file = "coverage-7.6.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f59d57baca39b32db42b83b2a7ba6f47ad9c394ec2076b084c3f029b7afca23"}, + {file = "coverage-7.6.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:a1ac0ae2b8bd743b88ed0502544847c3053d7171a3cff9228af618a068ed9c34"}, + {file = "coverage-7.6.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:e6a08c0be454c3b3beb105c0596ebdc2371fab6bb90c0c0297f4e58fd7e1012c"}, + {file = "coverage-7.6.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:f5796e664fe802da4f57a168c85359a8fbf3eab5e55cd4e4569fbacecc903959"}, + {file = "coverage-7.6.1-cp310-cp310-win32.whl", hash = "sha256:7bb65125fcbef8d989fa1dd0e8a060999497629ca5b0efbca209588a73356232"}, + {file = "coverage-7.6.1-cp310-cp310-win_amd64.whl", hash = "sha256:3115a95daa9bdba70aea750db7b96b37259a81a709223c8448fa97727d546fe0"}, + {file = "coverage-7.6.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:7dea0889685db8550f839fa202744652e87c60015029ce3f60e006f8c4462c93"}, + {file = "coverage-7.6.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ed37bd3c3b063412f7620464a9ac1314d33100329f39799255fb8d3027da50d3"}, + {file = "coverage-7.6.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d85f5e9a5f8b73e2350097c3756ef7e785f55bd71205defa0bfdaf96c31616ff"}, + {file = "coverage-7.6.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9bc572be474cafb617672c43fe989d6e48d3c83af02ce8de73fff1c6bb3c198d"}, + {file = "coverage-7.6.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0c0420b573964c760df9e9e86d1a9a622d0d27f417e1a949a8a66dd7bcee7bc6"}, + {file = "coverage-7.6.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1f4aa8219db826ce6be7099d559f8ec311549bfc4046f7f9fe9b5cea5c581c56"}, + {file = "coverage-7.6.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:fc5a77d0c516700ebad189b587de289a20a78324bc54baee03dd486f0855d234"}, + {file = "coverage-7.6.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b48f312cca9621272ae49008c7f613337c53fadca647d6384cc129d2996d1133"}, + {file = "coverage-7.6.1-cp311-cp311-win32.whl", hash = "sha256:1125ca0e5fd475cbbba3bb67ae20bd2c23a98fac4e32412883f9bcbaa81c314c"}, + {file = "coverage-7.6.1-cp311-cp311-win_amd64.whl", hash = "sha256:8ae539519c4c040c5ffd0632784e21b2f03fc1340752af711f33e5be83a9d6c6"}, + {file = "coverage-7.6.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:95cae0efeb032af8458fc27d191f85d1717b1d4e49f7cb226cf526ff28179778"}, + {file = "coverage-7.6.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5621a9175cf9d0b0c84c2ef2b12e9f5f5071357c4d2ea6ca1cf01814f45d2391"}, + {file = "coverage-7.6.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:260933720fdcd75340e7dbe9060655aff3af1f0c5d20f46b57f262ab6c86a5e8"}, + {file = "coverage-7.6.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:07e2ca0ad381b91350c0ed49d52699b625aab2b44b65e1b4e02fa9df0e92ad2d"}, + {file = "coverage-7.6.1-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c44fee9975f04b33331cb8eb272827111efc8930cfd582e0320613263ca849ca"}, + {file = "coverage-7.6.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:877abb17e6339d96bf08e7a622d05095e72b71f8afd8a9fefc82cf30ed944163"}, + {file = "coverage-7.6.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:3e0cadcf6733c09154b461f1ca72d5416635e5e4ec4e536192180d34ec160f8a"}, + {file = "coverage-7.6.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c3c02d12f837d9683e5ab2f3d9844dc57655b92c74e286c262e0fc54213c216d"}, + {file = "coverage-7.6.1-cp312-cp312-win32.whl", hash = "sha256:e05882b70b87a18d937ca6768ff33cc3f72847cbc4de4491c8e73880766718e5"}, + {file = "coverage-7.6.1-cp312-cp312-win_amd64.whl", hash = "sha256:b5d7b556859dd85f3a541db6a4e0167b86e7273e1cdc973e5b175166bb634fdb"}, + {file = "coverage-7.6.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:a4acd025ecc06185ba2b801f2de85546e0b8ac787cf9d3b06e7e2a69f925b106"}, + {file = "coverage-7.6.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a6d3adcf24b624a7b778533480e32434a39ad8fa30c315208f6d3e5542aeb6e9"}, + {file = "coverage-7.6.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d0c212c49b6c10e6951362f7c6df3329f04c2b1c28499563d4035d964ab8e08c"}, + {file = "coverage-7.6.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6e81d7a3e58882450ec4186ca59a3f20a5d4440f25b1cff6f0902ad890e6748a"}, + {file = "coverage-7.6.1-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:78b260de9790fd81e69401c2dc8b17da47c8038176a79092a89cb2b7d945d060"}, + {file = "coverage-7.6.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a78d169acd38300060b28d600344a803628c3fd585c912cacc9ea8790fe96862"}, + {file = "coverage-7.6.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:2c09f4ce52cb99dd7505cd0fc8e0e37c77b87f46bc9c1eb03fe3bc9991085388"}, + {file = "coverage-7.6.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6878ef48d4227aace338d88c48738a4258213cd7b74fd9a3d4d7582bb1d8a155"}, + {file = "coverage-7.6.1-cp313-cp313-win32.whl", hash = "sha256:44df346d5215a8c0e360307d46ffaabe0f5d3502c8a1cefd700b34baf31d411a"}, + {file = "coverage-7.6.1-cp313-cp313-win_amd64.whl", hash = "sha256:8284cf8c0dd272a247bc154eb6c95548722dce90d098c17a883ed36e67cdb129"}, + {file = "coverage-7.6.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:d3296782ca4eab572a1a4eca686d8bfb00226300dcefdf43faa25b5242ab8a3e"}, + {file = "coverage-7.6.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:502753043567491d3ff6d08629270127e0c31d4184c4c8d98f92c26f65019962"}, + {file = "coverage-7.6.1-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6a89ecca80709d4076b95f89f308544ec8f7b4727e8a547913a35f16717856cb"}, + {file = "coverage-7.6.1-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a318d68e92e80af8b00fa99609796fdbcdfef3629c77c6283566c6f02c6d6704"}, + {file = "coverage-7.6.1-cp313-cp313t-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:13b0a73a0896988f053e4fbb7de6d93388e6dd292b0d87ee51d106f2c11b465b"}, + {file = "coverage-7.6.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:4421712dbfc5562150f7554f13dde997a2e932a6b5f352edcce948a815efee6f"}, + {file = "coverage-7.6.1-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:166811d20dfea725e2e4baa71fffd6c968a958577848d2131f39b60043400223"}, + {file = "coverage-7.6.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:225667980479a17db1048cb2bf8bfb39b8e5be8f164b8f6628b64f78a72cf9d3"}, + {file = "coverage-7.6.1-cp313-cp313t-win32.whl", hash = "sha256:170d444ab405852903b7d04ea9ae9b98f98ab6d7e63e1115e82620807519797f"}, + {file = "coverage-7.6.1-cp313-cp313t-win_amd64.whl", hash = "sha256:b9f222de8cded79c49bf184bdbc06630d4c58eec9459b939b4a690c82ed05657"}, + {file = "coverage-7.6.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6db04803b6c7291985a761004e9060b2bca08da6d04f26a7f2294b8623a0c1a0"}, + {file = "coverage-7.6.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:f1adfc8ac319e1a348af294106bc6a8458a0f1633cc62a1446aebc30c5fa186a"}, + {file = "coverage-7.6.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a95324a9de9650a729239daea117df21f4b9868ce32e63f8b650ebe6cef5595b"}, + {file = "coverage-7.6.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b43c03669dc4618ec25270b06ecd3ee4fa94c7f9b3c14bae6571ca00ef98b0d3"}, + {file = "coverage-7.6.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8929543a7192c13d177b770008bc4e8119f2e1f881d563fc6b6305d2d0ebe9de"}, + {file = "coverage-7.6.1-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:a09ece4a69cf399510c8ab25e0950d9cf2b42f7b3cb0374f95d2e2ff594478a6"}, + {file = "coverage-7.6.1-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:9054a0754de38d9dbd01a46621636689124d666bad1936d76c0341f7d71bf569"}, + {file = "coverage-7.6.1-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:0dbde0f4aa9a16fa4d754356a8f2e36296ff4d83994b2c9d8398aa32f222f989"}, + {file = "coverage-7.6.1-cp38-cp38-win32.whl", hash = "sha256:da511e6ad4f7323ee5702e6633085fb76c2f893aaf8ce4c51a0ba4fc07580ea7"}, + {file = "coverage-7.6.1-cp38-cp38-win_amd64.whl", hash = "sha256:3f1156e3e8f2872197af3840d8ad307a9dd18e615dc64d9ee41696f287c57ad8"}, + {file = "coverage-7.6.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:abd5fd0db5f4dc9289408aaf34908072f805ff7792632250dcb36dc591d24255"}, + {file = "coverage-7.6.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:547f45fa1a93154bd82050a7f3cddbc1a7a4dd2a9bf5cb7d06f4ae29fe94eaf8"}, + {file = "coverage-7.6.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:645786266c8f18a931b65bfcefdbf6952dd0dea98feee39bd188607a9d307ed2"}, + {file = "coverage-7.6.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9e0b2df163b8ed01d515807af24f63de04bebcecbd6c3bfeff88385789fdf75a"}, + {file = "coverage-7.6.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:609b06f178fe8e9f89ef676532760ec0b4deea15e9969bf754b37f7c40326dbc"}, + {file = "coverage-7.6.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:702855feff378050ae4f741045e19a32d57d19f3e0676d589df0575008ea5004"}, + {file = "coverage-7.6.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:2bdb062ea438f22d99cba0d7829c2ef0af1d768d1e4a4f528087224c90b132cb"}, + {file = "coverage-7.6.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:9c56863d44bd1c4fe2abb8a4d6f5371d197f1ac0ebdee542f07f35895fc07f36"}, + {file = "coverage-7.6.1-cp39-cp39-win32.whl", hash = "sha256:6e2cd258d7d927d09493c8df1ce9174ad01b381d4729a9d8d4e38670ca24774c"}, + {file = "coverage-7.6.1-cp39-cp39-win_amd64.whl", hash = "sha256:06a737c882bd26d0d6ee7269b20b12f14a8704807a01056c80bb881a4b2ce6ca"}, + {file = "coverage-7.6.1-pp38.pp39.pp310-none-any.whl", hash = "sha256:e9a6e0eb86070e8ccaedfbd9d38fec54864f3125ab95419970575b42af7541df"}, + {file = "coverage-7.6.1.tar.gz", hash = "sha256:953510dfb7b12ab69d20135a0662397f077c59b1e6379a768e97c59d852ee51d"}, ] [package.dependencies] @@ -696,13 +716,13 @@ ovld = ">=0.3.6,<0.4.0" [[package]] name = "idna" -version = "3.7" +version = "3.8" description = "Internationalized Domain Names in Applications (IDNA)" optional = false -python-versions = ">=3.5" +python-versions = ">=3.6" files = [ - {file = "idna-3.7-py3-none-any.whl", hash = "sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0"}, - {file = "idna-3.7.tar.gz", hash = "sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc"}, + {file = "idna-3.8-py3-none-any.whl", hash = "sha256:050b4e5baadcd44d760cedbd2b8e639f2ff89bbc7a5730fcc662954303377aac"}, + {file = "idna-3.8.tar.gz", hash = "sha256:d838c2c0ed6fced7693d5e8ab8e734d5f8fda53a039c0164afb0b82e771e3603"}, ] [[package]] @@ -718,13 +738,13 @@ files = [ [[package]] name = "importlib-metadata" -version = "8.2.0" +version = "8.4.0" description = "Read metadata from Python packages" optional = false python-versions = ">=3.8" files = [ - {file = "importlib_metadata-8.2.0-py3-none-any.whl", hash = "sha256:11901fa0c2f97919b288679932bb64febaeacf289d18ac84dd68cb2e74213369"}, - {file = "importlib_metadata-8.2.0.tar.gz", hash = "sha256:72e8d4399996132204f9a16dcc751af254a48f8d1b20b9ff0f98d4a8f901e73d"}, + {file = "importlib_metadata-8.4.0-py3-none-any.whl", hash = "sha256:66f342cc6ac9818fc6ff340576acd24d65ba0b3efabb2b4ac08b598965a4a2f1"}, + {file = "importlib_metadata-8.4.0.tar.gz", hash = "sha256:9a547d3bc3608b025f93d403fdd1aae741c24fbb8314df4b155675742ce303c5"}, ] [package.dependencies] @@ -737,18 +757,22 @@ test = ["flufl.flake8", "importlib-resources (>=1.3)", "jaraco.test (>=5.4)", "p [[package]] name = "importlib-resources" -version = "6.4.0" +version = "6.4.4" description = "Read resources from Python packages" optional = false python-versions = ">=3.8" files = [ - {file = "importlib_resources-6.4.0-py3-none-any.whl", hash = "sha256:50d10f043df931902d4194ea07ec57960f66a80449ff867bfe782b4c486ba78c"}, - {file = "importlib_resources-6.4.0.tar.gz", hash = "sha256:cdb2b453b8046ca4e3798eb1d84f3cce1446a0e8e7b5ef4efb600f19fc398145"}, + {file = "importlib_resources-6.4.4-py3-none-any.whl", hash = "sha256:dda242603d1c9cd836c3368b1174ed74cb4049ecd209e7a1a0104620c18c5c11"}, + {file = "importlib_resources-6.4.4.tar.gz", hash = "sha256:20600c8b7361938dc0bb2d5ec0297802e575df486f5a544fa414da65e13721f7"}, ] [package.extras] -docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-lint"] -testing = ["jaraco.test (>=5.4)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy", "pytest-ruff (>=0.2.1)", "zipp (>=3.17)"] +check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1)"] +cover = ["pytest-cov"] +doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] +enabler = ["pytest-enabler (>=2.2)"] +test = ["jaraco.test (>=5.4)", "pytest (>=6,!=8.1.*)", "zipp (>=3.17)"] +type = ["pytest-mypy"] [[package]] name = "iniconfig" @@ -794,13 +818,13 @@ i18n = ["Babel (>=2.7)"] [[package]] name = "jinxed" -version = "1.2.1" +version = "1.3.0" description = "Jinxed Terminal Library" optional = false python-versions = "*" files = [ - {file = "jinxed-1.2.1-py2.py3-none-any.whl", hash = "sha256:37422659c4925969c66148c5e64979f553386a4226b9484d910d3094ced37d30"}, - {file = "jinxed-1.2.1.tar.gz", hash = "sha256:30c3f861b73279fea1ed928cfd4dfb1f273e16cd62c8a32acfac362da0f78f3f"}, + {file = "jinxed-1.3.0-py2.py3-none-any.whl", hash = "sha256:b993189f39dc2d7504d802152671535b06d380b26d78070559551cbf92df4fc5"}, + {file = "jinxed-1.3.0.tar.gz", hash = "sha256:1593124b18a41b7a3da3b078471442e51dbad3d77b4d4f2b0c26ab6f7d660dbf"}, ] [package.dependencies] @@ -1132,13 +1156,13 @@ files = [ [[package]] name = "pip" -version = "24.1.2" +version = "24.2" description = "The PyPA recommended tool for installing Python packages." optional = false python-versions = ">=3.8" files = [ - {file = "pip-24.1.2-py3-none-any.whl", hash = "sha256:7cd207eed4c60b0f411b444cd1464198fe186671c323b6cd6d433ed80fc9d247"}, - {file = "pip-24.1.2.tar.gz", hash = "sha256:e5458a0b89f2755e0ee8c0c77613fe5273e05f337907874d64f13171a898a7ff"}, + {file = "pip-24.2-py3-none-any.whl", hash = "sha256:2cd581cf58ab7fcfca4ce8efa6dcacd0de5bf8d0a3eb9ec927e07405f4d9e2a2"}, + {file = "pip-24.2.tar.gz", hash = "sha256:5b5e490b5e9cb275c879595064adce9ebd31b854e3e803740b72f9ccf34a45b8"}, ] [[package]] @@ -1582,62 +1606,64 @@ files = [ [[package]] name = "pyyaml" -version = "6.0.1" +version = "6.0.2" description = "YAML parser and emitter for Python" optional = false -python-versions = ">=3.6" +python-versions = ">=3.8" files = [ - {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, - {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, - {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, - {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, - {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, - {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, - {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, - {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, - {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, - {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, - {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, - {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, - {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, - {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, - {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, - {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, - {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, - {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, - {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, - {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, - {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, - {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, - {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, - {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, - {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, - {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, - {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, - {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, - {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, - {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, - {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, - {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, - {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, - {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, - {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, - {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, - {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, - {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, - {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, - {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, - {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, - {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, - {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, - {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, - {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, - {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, - {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, + {file = "PyYAML-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086"}, + {file = "PyYAML-6.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf"}, + {file = "PyYAML-6.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8824b5a04a04a047e72eea5cec3bc266db09e35de6bdfe34c9436ac5ee27d237"}, + {file = "PyYAML-6.0.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7c36280e6fb8385e520936c3cb3b8042851904eba0e58d277dca80a5cfed590b"}, + {file = "PyYAML-6.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ec031d5d2feb36d1d1a24380e4db6d43695f3748343d99434e6f5f9156aaa2ed"}, + {file = "PyYAML-6.0.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:936d68689298c36b53b29f23c6dbb74de12b4ac12ca6cfe0e047bedceea56180"}, + {file = "PyYAML-6.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:23502f431948090f597378482b4812b0caae32c22213aecf3b55325e049a6c68"}, + {file = "PyYAML-6.0.2-cp310-cp310-win32.whl", hash = "sha256:2e99c6826ffa974fe6e27cdb5ed0021786b03fc98e5ee3c5bfe1fd5015f42b99"}, + {file = "PyYAML-6.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:a4d3091415f010369ae4ed1fc6b79def9416358877534caf6a0fdd2146c87a3e"}, + {file = "PyYAML-6.0.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:cc1c1159b3d456576af7a3e4d1ba7e6924cb39de8f67111c735f6fc832082774"}, + {file = "PyYAML-6.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1e2120ef853f59c7419231f3bf4e7021f1b936f6ebd222406c3b60212205d2ee"}, + {file = "PyYAML-6.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5d225db5a45f21e78dd9358e58a98702a0302f2659a3c6cd320564b75b86f47c"}, + {file = "PyYAML-6.0.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5ac9328ec4831237bec75defaf839f7d4564be1e6b25ac710bd1a96321cc8317"}, + {file = "PyYAML-6.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ad2a3decf9aaba3d29c8f537ac4b243e36bef957511b4766cb0057d32b0be85"}, + {file = "PyYAML-6.0.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:ff3824dc5261f50c9b0dfb3be22b4567a6f938ccce4587b38952d85fd9e9afe4"}, + {file = "PyYAML-6.0.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:797b4f722ffa07cc8d62053e4cff1486fa6dc094105d13fea7b1de7d8bf71c9e"}, + {file = "PyYAML-6.0.2-cp311-cp311-win32.whl", hash = "sha256:11d8f3dd2b9c1207dcaf2ee0bbbfd5991f571186ec9cc78427ba5bd32afae4b5"}, + {file = "PyYAML-6.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:e10ce637b18caea04431ce14fabcf5c64a1c61ec9c56b071a4b7ca131ca52d44"}, + {file = "PyYAML-6.0.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:c70c95198c015b85feafc136515252a261a84561b7b1d51e3384e0655ddf25ab"}, + {file = "PyYAML-6.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ce826d6ef20b1bc864f0a68340c8b3287705cae2f8b4b1d932177dcc76721725"}, + {file = "PyYAML-6.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f71ea527786de97d1a0cc0eacd1defc0985dcf6b3f17bb77dcfc8c34bec4dc5"}, + {file = "PyYAML-6.0.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9b22676e8097e9e22e36d6b7bda33190d0d400f345f23d4065d48f4ca7ae0425"}, + {file = "PyYAML-6.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80bab7bfc629882493af4aa31a4cfa43a4c57c83813253626916b8c7ada83476"}, + {file = "PyYAML-6.0.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:0833f8694549e586547b576dcfaba4a6b55b9e96098b36cdc7ebefe667dfed48"}, + {file = "PyYAML-6.0.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8b9c7197f7cb2738065c481a0461e50ad02f18c78cd75775628afb4d7137fb3b"}, + {file = "PyYAML-6.0.2-cp312-cp312-win32.whl", hash = "sha256:ef6107725bd54b262d6dedcc2af448a266975032bc85ef0172c5f059da6325b4"}, + {file = "PyYAML-6.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:7e7401d0de89a9a855c839bc697c079a4af81cf878373abd7dc625847d25cbd8"}, + {file = "PyYAML-6.0.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:efdca5630322a10774e8e98e1af481aad470dd62c3170801852d752aa7a783ba"}, + {file = "PyYAML-6.0.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:50187695423ffe49e2deacb8cd10510bc361faac997de9efef88badc3bb9e2d1"}, + {file = "PyYAML-6.0.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0ffe8360bab4910ef1b9e87fb812d8bc0a308b0d0eef8c8f44e0254ab3b07133"}, + {file = "PyYAML-6.0.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:17e311b6c678207928d649faa7cb0d7b4c26a0ba73d41e99c4fff6b6c3276484"}, + {file = "PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:70b189594dbe54f75ab3a1acec5f1e3faa7e8cf2f1e08d9b561cb41b845f69d5"}, + {file = "PyYAML-6.0.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:41e4e3953a79407c794916fa277a82531dd93aad34e29c2a514c2c0c5fe971cc"}, + {file = "PyYAML-6.0.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:68ccc6023a3400877818152ad9a1033e3db8625d899c72eacb5a668902e4d652"}, + {file = "PyYAML-6.0.2-cp313-cp313-win32.whl", hash = "sha256:bc2fa7c6b47d6bc618dd7fb02ef6fdedb1090ec036abab80d4681424b84c1183"}, + {file = "PyYAML-6.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:8388ee1976c416731879ac16da0aff3f63b286ffdd57cdeb95f3f2e085687563"}, + {file = "PyYAML-6.0.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:24471b829b3bf607e04e88d79542a9d48bb037c2267d7927a874e6c205ca7e9a"}, + {file = "PyYAML-6.0.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d7fded462629cfa4b685c5416b949ebad6cec74af5e2d42905d41e257e0869f5"}, + {file = "PyYAML-6.0.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d84a1718ee396f54f3a086ea0a66d8e552b2ab2017ef8b420e92edbc841c352d"}, + {file = "PyYAML-6.0.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9056c1ecd25795207ad294bcf39f2db3d845767be0ea6e6a34d856f006006083"}, + {file = "PyYAML-6.0.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:82d09873e40955485746739bcb8b4586983670466c23382c19cffecbf1fd8706"}, + {file = "PyYAML-6.0.2-cp38-cp38-win32.whl", hash = "sha256:43fa96a3ca0d6b1812e01ced1044a003533c47f6ee8aca31724f78e93ccc089a"}, + {file = "PyYAML-6.0.2-cp38-cp38-win_amd64.whl", hash = "sha256:01179a4a8559ab5de078078f37e5c1a30d76bb88519906844fd7bdea1b7729ff"}, + {file = "PyYAML-6.0.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:688ba32a1cffef67fd2e9398a2efebaea461578b0923624778664cc1c914db5d"}, + {file = "PyYAML-6.0.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a8786accb172bd8afb8be14490a16625cbc387036876ab6ba70912730faf8e1f"}, + {file = "PyYAML-6.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8e03406cac8513435335dbab54c0d385e4a49e4945d2909a581c83647ca0290"}, + {file = "PyYAML-6.0.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f753120cb8181e736c57ef7636e83f31b9c0d1722c516f7e86cf15b7aa57ff12"}, + {file = "PyYAML-6.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3b1fdb9dc17f5a7677423d508ab4f243a726dea51fa5e70992e59a7411c89d19"}, + {file = "PyYAML-6.0.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:0b69e4ce7a131fe56b7e4d770c67429700908fc0752af059838b1cfb41960e4e"}, + {file = "PyYAML-6.0.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a9f8c2e67970f13b16084e04f134610fd1d374bf477b17ec1599185cf611d725"}, + {file = "PyYAML-6.0.2-cp39-cp39-win32.whl", hash = "sha256:6395c297d42274772abc367baaa79683958044e5d3835486c16da75d2a694631"}, + {file = "PyYAML-6.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:39693e1f8320ae4f43943590b49779ffb98acb81f788220ea932a6b6c51004d8"}, + {file = "pyyaml-6.0.2.tar.gz", hash = "sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e"}, ] [[package]] @@ -1677,13 +1703,13 @@ use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] [[package]] name = "rich" -version = "13.7.1" +version = "13.8.0" description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" optional = false python-versions = ">=3.7.0" files = [ - {file = "rich-13.7.1-py3-none-any.whl", hash = "sha256:4edbae314f59eb482f54e9e30bf00d33350aaa94f4bfcd4e9e3110e64d0d7222"}, - {file = "rich-13.7.1.tar.gz", hash = "sha256:9be308cb1fe2f1f57d67ce99e95af38a1e2bc71ad9813b0e247cf7ffbcc3a432"}, + {file = "rich-13.8.0-py3-none-any.whl", hash = "sha256:2e85306a063b9492dffc86278197a60cbece75bcb766022f3436f567cae11bdc"}, + {file = "rich-13.8.0.tar.gz", hash = "sha256:a5ac1f1cd448ade0d59cc3356f7db7a7ccda2c8cbae9c7a90c28ff463d3e91f4"}, ] [package.dependencies] @@ -1695,19 +1721,19 @@ jupyter = ["ipywidgets (>=7.5.1,<9)"] [[package]] name = "setuptools" -version = "71.1.0" +version = "73.0.1" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false python-versions = ">=3.8" files = [ - {file = "setuptools-71.1.0-py3-none-any.whl", hash = "sha256:33874fdc59b3188304b2e7c80d9029097ea31627180896fb549c578ceb8a0855"}, - {file = "setuptools-71.1.0.tar.gz", hash = "sha256:032d42ee9fb536e33087fb66cac5f840eb9391ed05637b3f2a76a7c8fb477936"}, + {file = "setuptools-73.0.1-py3-none-any.whl", hash = "sha256:b208925fcb9f7af924ed2dc04708ea89791e24bde0d3020b27df0e116088b34e"}, + {file = "setuptools-73.0.1.tar.gz", hash = "sha256:d59a3e788ab7e012ab2c4baed1b376da6366883ee20d7a5fc426816e3d7b1193"}, ] [package.extras] -core = ["importlib-metadata (>=6)", "importlib-resources (>=5.10.2)", "jaraco.text (>=3.7)", "more-itertools (>=8.8)", "ordered-set (>=3.1.1)", "packaging (>=24)", "platformdirs (>=2.6.2)", "tomli (>=2.0.1)", "wheel (>=0.43.0)"] -doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] -test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "jaraco.test", "mypy (==1.11.*)", "packaging (>=23.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy", "pytest-perf", "pytest-ruff (<0.4)", "pytest-ruff (>=0.2.1)", "pytest-ruff (>=0.3.2)", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +core = ["importlib-metadata (>=6)", "importlib-resources (>=5.10.2)", "jaraco.text (>=3.7)", "more-itertools (>=8.8)", "packaging (>=24)", "platformdirs (>=2.6.2)", "tomli (>=2.0.1)", "wheel (>=0.43.0)"] +doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier", "towncrier (<24.7)"] +test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "jaraco.test", "mypy (==1.11.*)", "packaging (>=23.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy", "pytest-perf", "pytest-ruff (<0.4)", "pytest-ruff (>=0.2.1)", "pytest-ruff (>=0.3.2)", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel (>=0.44.0)"] [[package]] name = "six" @@ -1797,49 +1823,49 @@ dev = ["bump2version", "sphinxcontrib-httpdomain", "transifex-client", "wheel"] [[package]] name = "sphinxcontrib-applehelp" -version = "1.0.8" +version = "2.0.0" description = "sphinxcontrib-applehelp is a Sphinx extension which outputs Apple help books" optional = false python-versions = ">=3.9" files = [ - {file = "sphinxcontrib_applehelp-1.0.8-py3-none-any.whl", hash = "sha256:cb61eb0ec1b61f349e5cc36b2028e9e7ca765be05e49641c97241274753067b4"}, - {file = "sphinxcontrib_applehelp-1.0.8.tar.gz", hash = "sha256:c40a4f96f3776c4393d933412053962fac2b84f4c99a7982ba42e09576a70619"}, + {file = "sphinxcontrib_applehelp-2.0.0-py3-none-any.whl", hash = "sha256:4cd3f0ec4ac5dd9c17ec65e9ab272c9b867ea77425228e68ecf08d6b28ddbdb5"}, + {file = "sphinxcontrib_applehelp-2.0.0.tar.gz", hash = "sha256:2f29ef331735ce958efa4734873f084941970894c6090408b079c61b2e1c06d1"}, ] [package.extras] -lint = ["docutils-stubs", "flake8", "mypy"] +lint = ["mypy", "ruff (==0.5.5)", "types-docutils"] standalone = ["Sphinx (>=5)"] test = ["pytest"] [[package]] name = "sphinxcontrib-devhelp" -version = "1.0.6" +version = "2.0.0" description = "sphinxcontrib-devhelp is a sphinx extension which outputs Devhelp documents" optional = false python-versions = ">=3.9" files = [ - {file = "sphinxcontrib_devhelp-1.0.6-py3-none-any.whl", hash = "sha256:6485d09629944511c893fa11355bda18b742b83a2b181f9a009f7e500595c90f"}, - {file = "sphinxcontrib_devhelp-1.0.6.tar.gz", hash = "sha256:9893fd3f90506bc4b97bdb977ceb8fbd823989f4316b28c3841ec128544372d3"}, + {file = "sphinxcontrib_devhelp-2.0.0-py3-none-any.whl", hash = "sha256:aefb8b83854e4b0998877524d1029fd3e6879210422ee3780459e28a1f03a8a2"}, + {file = "sphinxcontrib_devhelp-2.0.0.tar.gz", hash = "sha256:411f5d96d445d1d73bb5d52133377b4248ec79db5c793ce7dbe59e074b4dd1ad"}, ] [package.extras] -lint = ["docutils-stubs", "flake8", "mypy"] +lint = ["mypy", "ruff (==0.5.5)", "types-docutils"] standalone = ["Sphinx (>=5)"] test = ["pytest"] [[package]] name = "sphinxcontrib-htmlhelp" -version = "2.0.6" +version = "2.1.0" description = "sphinxcontrib-htmlhelp is a sphinx extension which renders HTML help files" optional = false python-versions = ">=3.9" files = [ - {file = "sphinxcontrib_htmlhelp-2.0.6-py3-none-any.whl", hash = "sha256:1b9af5a2671a61410a868fce050cab7ca393c218e6205cbc7f590136f207395c"}, - {file = "sphinxcontrib_htmlhelp-2.0.6.tar.gz", hash = "sha256:c6597da06185f0e3b4dc952777a04200611ef563882e0c244d27a15ee22afa73"}, + {file = "sphinxcontrib_htmlhelp-2.1.0-py3-none-any.whl", hash = "sha256:166759820b47002d22914d64a075ce08f4c46818e17cfc9470a9786b759b19f8"}, + {file = "sphinxcontrib_htmlhelp-2.1.0.tar.gz", hash = "sha256:c9e2916ace8aad64cc13a0d233ee22317f2b9025b9cf3295249fa985cc7082e9"}, ] [package.extras] -lint = ["docutils-stubs", "flake8", "mypy"] +lint = ["mypy", "ruff (==0.5.5)", "types-docutils"] standalone = ["Sphinx (>=5)"] test = ["html5lib", "pytest"] @@ -1873,92 +1899,92 @@ test = ["flake8", "mypy", "pytest"] [[package]] name = "sphinxcontrib-qthelp" -version = "1.0.8" +version = "2.0.0" description = "sphinxcontrib-qthelp is a sphinx extension which outputs QtHelp documents" optional = false python-versions = ">=3.9" files = [ - {file = "sphinxcontrib_qthelp-1.0.8-py3-none-any.whl", hash = "sha256:323d6acc4189af76dfe94edd2a27d458902319b60fcca2aeef3b2180c106a75f"}, - {file = "sphinxcontrib_qthelp-1.0.8.tar.gz", hash = "sha256:db3f8fa10789c7a8e76d173c23364bdf0ebcd9449969a9e6a3dd31b8b7469f03"}, + {file = "sphinxcontrib_qthelp-2.0.0-py3-none-any.whl", hash = "sha256:b18a828cdba941ccd6ee8445dbe72ffa3ef8cbe7505d8cd1fa0d42d3f2d5f3eb"}, + {file = "sphinxcontrib_qthelp-2.0.0.tar.gz", hash = "sha256:4fe7d0ac8fc171045be623aba3e2a8f613f8682731f9153bb2e40ece16b9bbab"}, ] [package.extras] -lint = ["docutils-stubs", "flake8", "mypy"] +lint = ["mypy", "ruff (==0.5.5)", "types-docutils"] standalone = ["Sphinx (>=5)"] test = ["defusedxml (>=0.7.1)", "pytest"] [[package]] name = "sphinxcontrib-serializinghtml" -version = "1.1.10" +version = "2.0.0" description = "sphinxcontrib-serializinghtml is a sphinx extension which outputs \"serialized\" HTML files (json and pickle)" optional = false python-versions = ">=3.9" files = [ - {file = "sphinxcontrib_serializinghtml-1.1.10-py3-none-any.whl", hash = "sha256:326369b8df80a7d2d8d7f99aa5ac577f51ea51556ed974e7716cfd4fca3f6cb7"}, - {file = "sphinxcontrib_serializinghtml-1.1.10.tar.gz", hash = "sha256:93f3f5dc458b91b192fe10c397e324f262cf163d79f3282c158e8436a2c4511f"}, + {file = "sphinxcontrib_serializinghtml-2.0.0-py3-none-any.whl", hash = "sha256:6e2cb0eef194e10c27ec0023bfeb25badbbb5868244cf5bc5bdc04e4464bf331"}, + {file = "sphinxcontrib_serializinghtml-2.0.0.tar.gz", hash = "sha256:e9d912827f872c029017a53f0ef2180b327c3f7fd23c87229f7a8e8b70031d4d"}, ] [package.extras] -lint = ["docutils-stubs", "flake8", "mypy"] +lint = ["mypy", "ruff (==0.5.5)", "types-docutils"] standalone = ["Sphinx (>=5)"] test = ["pytest"] [[package]] name = "sqlalchemy" -version = "2.0.31" +version = "2.0.32" description = "Database Abstraction Library" optional = false python-versions = ">=3.7" files = [ - {file = "SQLAlchemy-2.0.31-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f2a213c1b699d3f5768a7272de720387ae0122f1becf0901ed6eaa1abd1baf6c"}, - {file = "SQLAlchemy-2.0.31-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9fea3d0884e82d1e33226935dac990b967bef21315cbcc894605db3441347443"}, - {file = "SQLAlchemy-2.0.31-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f3ad7f221d8a69d32d197e5968d798217a4feebe30144986af71ada8c548e9fa"}, - {file = "SQLAlchemy-2.0.31-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9f2bee229715b6366f86a95d497c347c22ddffa2c7c96143b59a2aa5cc9eebbc"}, - {file = "SQLAlchemy-2.0.31-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:cd5b94d4819c0c89280b7c6109c7b788a576084bf0a480ae17c227b0bc41e109"}, - {file = "SQLAlchemy-2.0.31-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:750900a471d39a7eeba57580b11983030517a1f512c2cb287d5ad0fcf3aebd58"}, - {file = "SQLAlchemy-2.0.31-cp310-cp310-win32.whl", hash = "sha256:7bd112be780928c7f493c1a192cd8c5fc2a2a7b52b790bc5a84203fb4381c6be"}, - {file = "SQLAlchemy-2.0.31-cp310-cp310-win_amd64.whl", hash = "sha256:5a48ac4d359f058474fadc2115f78a5cdac9988d4f99eae44917f36aa1476327"}, - {file = "SQLAlchemy-2.0.31-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f68470edd70c3ac3b6cd5c2a22a8daf18415203ca1b036aaeb9b0fb6f54e8298"}, - {file = "SQLAlchemy-2.0.31-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2e2c38c2a4c5c634fe6c3c58a789712719fa1bf9b9d6ff5ebfce9a9e5b89c1ca"}, - {file = "SQLAlchemy-2.0.31-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bd15026f77420eb2b324dcb93551ad9c5f22fab2c150c286ef1dc1160f110203"}, - {file = "SQLAlchemy-2.0.31-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2196208432deebdfe3b22185d46b08f00ac9d7b01284e168c212919891289396"}, - {file = "SQLAlchemy-2.0.31-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:352b2770097f41bff6029b280c0e03b217c2dcaddc40726f8f53ed58d8a85da4"}, - {file = "SQLAlchemy-2.0.31-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:56d51ae825d20d604583f82c9527d285e9e6d14f9a5516463d9705dab20c3740"}, - {file = "SQLAlchemy-2.0.31-cp311-cp311-win32.whl", hash = "sha256:6e2622844551945db81c26a02f27d94145b561f9d4b0c39ce7bfd2fda5776dac"}, - {file = "SQLAlchemy-2.0.31-cp311-cp311-win_amd64.whl", hash = "sha256:ccaf1b0c90435b6e430f5dd30a5aede4764942a695552eb3a4ab74ed63c5b8d3"}, - {file = "SQLAlchemy-2.0.31-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3b74570d99126992d4b0f91fb87c586a574a5872651185de8297c6f90055ae42"}, - {file = "SQLAlchemy-2.0.31-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6f77c4f042ad493cb8595e2f503c7a4fe44cd7bd59c7582fd6d78d7e7b8ec52c"}, - {file = "SQLAlchemy-2.0.31-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cd1591329333daf94467e699e11015d9c944f44c94d2091f4ac493ced0119449"}, - {file = "SQLAlchemy-2.0.31-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:74afabeeff415e35525bf7a4ecdab015f00e06456166a2eba7590e49f8db940e"}, - {file = "SQLAlchemy-2.0.31-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:b9c01990d9015df2c6f818aa8f4297d42ee71c9502026bb074e713d496e26b67"}, - {file = "SQLAlchemy-2.0.31-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:66f63278db425838b3c2b1c596654b31939427016ba030e951b292e32b99553e"}, - {file = "SQLAlchemy-2.0.31-cp312-cp312-win32.whl", hash = "sha256:0b0f658414ee4e4b8cbcd4a9bb0fd743c5eeb81fc858ca517217a8013d282c96"}, - {file = "SQLAlchemy-2.0.31-cp312-cp312-win_amd64.whl", hash = "sha256:fa4b1af3e619b5b0b435e333f3967612db06351217c58bfb50cee5f003db2a5a"}, - {file = "SQLAlchemy-2.0.31-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:f43e93057cf52a227eda401251c72b6fbe4756f35fa6bfebb5d73b86881e59b0"}, - {file = "SQLAlchemy-2.0.31-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d337bf94052856d1b330d5fcad44582a30c532a2463776e1651bd3294ee7e58b"}, - {file = "SQLAlchemy-2.0.31-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c06fb43a51ccdff3b4006aafee9fcf15f63f23c580675f7734245ceb6b6a9e05"}, - {file = "SQLAlchemy-2.0.31-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:b6e22630e89f0e8c12332b2b4c282cb01cf4da0d26795b7eae16702a608e7ca1"}, - {file = "SQLAlchemy-2.0.31-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:79a40771363c5e9f3a77f0e28b3302801db08040928146e6808b5b7a40749c88"}, - {file = "SQLAlchemy-2.0.31-cp37-cp37m-win32.whl", hash = "sha256:501ff052229cb79dd4c49c402f6cb03b5a40ae4771efc8bb2bfac9f6c3d3508f"}, - {file = "SQLAlchemy-2.0.31-cp37-cp37m-win_amd64.whl", hash = "sha256:597fec37c382a5442ffd471f66ce12d07d91b281fd474289356b1a0041bdf31d"}, - {file = "SQLAlchemy-2.0.31-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:dc6d69f8829712a4fd799d2ac8d79bdeff651c2301b081fd5d3fe697bd5b4ab9"}, - {file = "SQLAlchemy-2.0.31-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:23b9fbb2f5dd9e630db70fbe47d963c7779e9c81830869bd7d137c2dc1ad05fb"}, - {file = "SQLAlchemy-2.0.31-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2a21c97efcbb9f255d5c12a96ae14da873233597dfd00a3a0c4ce5b3e5e79704"}, - {file = "SQLAlchemy-2.0.31-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:26a6a9837589c42b16693cf7bf836f5d42218f44d198f9343dd71d3164ceeeac"}, - {file = "SQLAlchemy-2.0.31-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:dc251477eae03c20fae8db9c1c23ea2ebc47331bcd73927cdcaecd02af98d3c3"}, - {file = "SQLAlchemy-2.0.31-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:2fd17e3bb8058359fa61248c52c7b09a97cf3c820e54207a50af529876451808"}, - {file = "SQLAlchemy-2.0.31-cp38-cp38-win32.whl", hash = "sha256:c76c81c52e1e08f12f4b6a07af2b96b9b15ea67ccdd40ae17019f1c373faa227"}, - {file = "SQLAlchemy-2.0.31-cp38-cp38-win_amd64.whl", hash = "sha256:4b600e9a212ed59355813becbcf282cfda5c93678e15c25a0ef896b354423238"}, - {file = "SQLAlchemy-2.0.31-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b6cf796d9fcc9b37011d3f9936189b3c8074a02a4ed0c0fbbc126772c31a6d4"}, - {file = "SQLAlchemy-2.0.31-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:78fe11dbe37d92667c2c6e74379f75746dc947ee505555a0197cfba9a6d4f1a4"}, - {file = "SQLAlchemy-2.0.31-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2fc47dc6185a83c8100b37acda27658fe4dbd33b7d5e7324111f6521008ab4fe"}, - {file = "SQLAlchemy-2.0.31-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8a41514c1a779e2aa9a19f67aaadeb5cbddf0b2b508843fcd7bafdf4c6864005"}, - {file = "SQLAlchemy-2.0.31-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:afb6dde6c11ea4525318e279cd93c8734b795ac8bb5dda0eedd9ebaca7fa23f1"}, - {file = "SQLAlchemy-2.0.31-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:3f9faef422cfbb8fd53716cd14ba95e2ef655400235c3dfad1b5f467ba179c8c"}, - {file = "SQLAlchemy-2.0.31-cp39-cp39-win32.whl", hash = "sha256:fc6b14e8602f59c6ba893980bea96571dd0ed83d8ebb9c4479d9ed5425d562e9"}, - {file = "SQLAlchemy-2.0.31-cp39-cp39-win_amd64.whl", hash = "sha256:3cb8a66b167b033ec72c3812ffc8441d4e9f5f78f5e31e54dcd4c90a4ca5bebc"}, - {file = "SQLAlchemy-2.0.31-py3-none-any.whl", hash = "sha256:69f3e3c08867a8e4856e92d7afb618b95cdee18e0bc1647b77599722c9a28911"}, - {file = "SQLAlchemy-2.0.31.tar.gz", hash = "sha256:b607489dd4a54de56984a0c7656247504bd5523d9d0ba799aef59d4add009484"}, + {file = "SQLAlchemy-2.0.32-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0c9045ecc2e4db59bfc97b20516dfdf8e41d910ac6fb667ebd3a79ea54084619"}, + {file = "SQLAlchemy-2.0.32-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:1467940318e4a860afd546ef61fefb98a14d935cd6817ed07a228c7f7c62f389"}, + {file = "SQLAlchemy-2.0.32-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5954463675cb15db8d4b521f3566a017c8789222b8316b1e6934c811018ee08b"}, + {file = "SQLAlchemy-2.0.32-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:167e7497035c303ae50651b351c28dc22a40bb98fbdb8468cdc971821b1ae533"}, + {file = "SQLAlchemy-2.0.32-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b27dfb676ac02529fb6e343b3a482303f16e6bc3a4d868b73935b8792edb52d0"}, + {file = "SQLAlchemy-2.0.32-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:bf2360a5e0f7bd75fa80431bf8ebcfb920c9f885e7956c7efde89031695cafb8"}, + {file = "SQLAlchemy-2.0.32-cp310-cp310-win32.whl", hash = "sha256:306fe44e754a91cd9d600a6b070c1f2fadbb4a1a257b8781ccf33c7067fd3e4d"}, + {file = "SQLAlchemy-2.0.32-cp310-cp310-win_amd64.whl", hash = "sha256:99db65e6f3ab42e06c318f15c98f59a436f1c78179e6a6f40f529c8cc7100b22"}, + {file = "SQLAlchemy-2.0.32-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:21b053be28a8a414f2ddd401f1be8361e41032d2ef5884b2f31d31cb723e559f"}, + {file = "SQLAlchemy-2.0.32-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:b178e875a7a25b5938b53b006598ee7645172fccafe1c291a706e93f48499ff5"}, + {file = "SQLAlchemy-2.0.32-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:723a40ee2cc7ea653645bd4cf024326dea2076673fc9d3d33f20f6c81db83e1d"}, + {file = "SQLAlchemy-2.0.32-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:295ff8689544f7ee7e819529633d058bd458c1fd7f7e3eebd0f9268ebc56c2a0"}, + {file = "SQLAlchemy-2.0.32-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:49496b68cd190a147118af585173ee624114dfb2e0297558c460ad7495f9dfe2"}, + {file = "SQLAlchemy-2.0.32-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:acd9b73c5c15f0ec5ce18128b1fe9157ddd0044abc373e6ecd5ba376a7e5d961"}, + {file = "SQLAlchemy-2.0.32-cp311-cp311-win32.whl", hash = "sha256:9365a3da32dabd3e69e06b972b1ffb0c89668994c7e8e75ce21d3e5e69ddef28"}, + {file = "SQLAlchemy-2.0.32-cp311-cp311-win_amd64.whl", hash = "sha256:8bd63d051f4f313b102a2af1cbc8b80f061bf78f3d5bd0843ff70b5859e27924"}, + {file = "SQLAlchemy-2.0.32-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:6bab3db192a0c35e3c9d1560eb8332463e29e5507dbd822e29a0a3c48c0a8d92"}, + {file = "SQLAlchemy-2.0.32-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:19d98f4f58b13900d8dec4ed09dd09ef292208ee44cc9c2fe01c1f0a2fe440e9"}, + {file = "SQLAlchemy-2.0.32-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3cd33c61513cb1b7371fd40cf221256456d26a56284e7d19d1f0b9f1eb7dd7e8"}, + {file = "SQLAlchemy-2.0.32-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7d6ba0497c1d066dd004e0f02a92426ca2df20fac08728d03f67f6960271feec"}, + {file = "SQLAlchemy-2.0.32-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2b6be53e4fde0065524f1a0a7929b10e9280987b320716c1509478b712a7688c"}, + {file = "SQLAlchemy-2.0.32-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:916a798f62f410c0b80b63683c8061f5ebe237b0f4ad778739304253353bc1cb"}, + {file = "SQLAlchemy-2.0.32-cp312-cp312-win32.whl", hash = "sha256:31983018b74908ebc6c996a16ad3690301a23befb643093fcfe85efd292e384d"}, + {file = "SQLAlchemy-2.0.32-cp312-cp312-win_amd64.whl", hash = "sha256:4363ed245a6231f2e2957cccdda3c776265a75851f4753c60f3004b90e69bfeb"}, + {file = "SQLAlchemy-2.0.32-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b8afd5b26570bf41c35c0121801479958b4446751a3971fb9a480c1afd85558e"}, + {file = "SQLAlchemy-2.0.32-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c750987fc876813f27b60d619b987b057eb4896b81117f73bb8d9918c14f1cad"}, + {file = "SQLAlchemy-2.0.32-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ada0102afff4890f651ed91120c1120065663506b760da4e7823913ebd3258be"}, + {file = "SQLAlchemy-2.0.32-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:78c03d0f8a5ab4f3034c0e8482cfcc415a3ec6193491cfa1c643ed707d476f16"}, + {file = "SQLAlchemy-2.0.32-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:3bd1cae7519283ff525e64645ebd7a3e0283f3c038f461ecc1c7b040a0c932a1"}, + {file = "SQLAlchemy-2.0.32-cp37-cp37m-win32.whl", hash = "sha256:01438ebcdc566d58c93af0171c74ec28efe6a29184b773e378a385e6215389da"}, + {file = "SQLAlchemy-2.0.32-cp37-cp37m-win_amd64.whl", hash = "sha256:4979dc80fbbc9d2ef569e71e0896990bc94df2b9fdbd878290bd129b65ab579c"}, + {file = "SQLAlchemy-2.0.32-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c742be912f57586ac43af38b3848f7688863a403dfb220193a882ea60e1ec3a"}, + {file = "SQLAlchemy-2.0.32-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:62e23d0ac103bcf1c5555b6c88c114089587bc64d048fef5bbdb58dfd26f96da"}, + {file = "SQLAlchemy-2.0.32-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:251f0d1108aab8ea7b9aadbd07fb47fb8e3a5838dde34aa95a3349876b5a1f1d"}, + {file = "SQLAlchemy-2.0.32-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0ef18a84e5116340e38eca3e7f9eeaaef62738891422e7c2a0b80feab165905f"}, + {file = "SQLAlchemy-2.0.32-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:3eb6a97a1d39976f360b10ff208c73afb6a4de86dd2a6212ddf65c4a6a2347d5"}, + {file = "SQLAlchemy-2.0.32-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:0c1c9b673d21477cec17ab10bc4decb1322843ba35b481585facd88203754fc5"}, + {file = "SQLAlchemy-2.0.32-cp38-cp38-win32.whl", hash = "sha256:c41a2b9ca80ee555decc605bd3c4520cc6fef9abde8fd66b1cf65126a6922d65"}, + {file = "SQLAlchemy-2.0.32-cp38-cp38-win_amd64.whl", hash = "sha256:8a37e4d265033c897892279e8adf505c8b6b4075f2b40d77afb31f7185cd6ecd"}, + {file = "SQLAlchemy-2.0.32-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:52fec964fba2ef46476312a03ec8c425956b05c20220a1a03703537824b5e8e1"}, + {file = "SQLAlchemy-2.0.32-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:328429aecaba2aee3d71e11f2477c14eec5990fb6d0e884107935f7fb6001632"}, + {file = "SQLAlchemy-2.0.32-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:85a01b5599e790e76ac3fe3aa2f26e1feba56270023d6afd5550ed63c68552b3"}, + {file = "SQLAlchemy-2.0.32-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aaf04784797dcdf4c0aa952c8d234fa01974c4729db55c45732520ce12dd95b4"}, + {file = "SQLAlchemy-2.0.32-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:4488120becf9b71b3ac718f4138269a6be99a42fe023ec457896ba4f80749525"}, + {file = "SQLAlchemy-2.0.32-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:14e09e083a5796d513918a66f3d6aedbc131e39e80875afe81d98a03312889e6"}, + {file = "SQLAlchemy-2.0.32-cp39-cp39-win32.whl", hash = "sha256:0d322cc9c9b2154ba7e82f7bf25ecc7c36fbe2d82e2933b3642fc095a52cfc78"}, + {file = "SQLAlchemy-2.0.32-cp39-cp39-win_amd64.whl", hash = "sha256:7dd8583df2f98dea28b5cd53a1beac963f4f9d087888d75f22fcc93a07cf8d84"}, + {file = "SQLAlchemy-2.0.32-py3-none-any.whl", hash = "sha256:e567a8793a692451f706b363ccf3c45e056b67d90ead58c3bc9471af5d212202"}, + {file = "SQLAlchemy-2.0.32.tar.gz", hash = "sha256:c1b88cc8b02b6a5f0efb0345a03672d4c897dc7d92585176f88c67346f565ea8"}, ] [package.dependencies] @@ -2033,13 +2059,13 @@ plugins = ["importlib-resources"] [[package]] name = "tqdm" -version = "4.66.4" +version = "4.66.5" description = "Fast, Extensible Progress Meter" optional = false python-versions = ">=3.7" files = [ - {file = "tqdm-4.66.4-py3-none-any.whl", hash = "sha256:b75ca56b413b030bc3f00af51fd2c1a1a5eac6a0c1cca83cbb37a5c52abce644"}, - {file = "tqdm-4.66.4.tar.gz", hash = "sha256:e4d936c9de8727928f3be6079590e97d9abfe8d39a590be678eb5919ffc186bb"}, + {file = "tqdm-4.66.5-py3-none-any.whl", hash = "sha256:90279a3770753eafc9194a0364852159802111925aa30eb3f9d85b0e805ac7cd"}, + {file = "tqdm-4.66.5.tar.gz", hash = "sha256:e1020aef2e5096702d8a025ac7d16b1577279c9d63f8375b63083e9a5f0fcbad"}, ] [package.dependencies] @@ -2160,13 +2186,13 @@ files = [ [[package]] name = "wheel" -version = "0.43.0" +version = "0.44.0" description = "A built-package format for Python" optional = false python-versions = ">=3.8" files = [ - {file = "wheel-0.43.0-py3-none-any.whl", hash = "sha256:55c570405f142630c6b9f72fe09d9b67cf1477fcf543ae5b8dcb1f5b7377da81"}, - {file = "wheel-0.43.0.tar.gz", hash = "sha256:465ef92c69fa5c5da2d1cf8ac40559a8c940886afcef87dcf14b9470862f1d85"}, + {file = "wheel-0.44.0-py3-none-any.whl", hash = "sha256:2376a90c98cc337d18623527a97c31797bd02bad0033d41547043a1cbfbe448f"}, + {file = "wheel-0.44.0.tar.gz", hash = "sha256:a29c3f2817e95ab89aa4660681ad547c0e9547f20e75b0562fe7723c9a2a9d49"}, ] [package.extras] @@ -2174,18 +2200,22 @@ test = ["pytest (>=6.0.0)", "setuptools (>=65)"] [[package]] name = "zipp" -version = "3.19.2" +version = "3.20.1" description = "Backport of pathlib-compatible object wrapper for zip files" optional = false python-versions = ">=3.8" files = [ - {file = "zipp-3.19.2-py3-none-any.whl", hash = "sha256:f091755f667055f2d02b32c53771a7a6c8b47e1fdbc4b72a8b9072b3eef8015c"}, - {file = "zipp-3.19.2.tar.gz", hash = "sha256:bf1dcf6450f873a13e952a29504887c89e6de7506209e5b1bcc3460135d4de19"}, + {file = "zipp-3.20.1-py3-none-any.whl", hash = "sha256:9960cd8967c8f85a56f920d5d507274e74f9ff813a0ab8889a5b5be2daf44064"}, + {file = "zipp-3.20.1.tar.gz", hash = "sha256:c22b14cc4763c5a5b04134207736c107db42e9d3ef2d9779d465f5f1bcba572b"}, ] [package.extras] +check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1)"] +cover = ["pytest-cov"] doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] -test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more-itertools", "pytest (>=6,!=8.1.*)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-ignore-flaky", "pytest-mypy", "pytest-ruff (>=0.2.1)"] +enabler = ["pytest-enabler (>=2.2)"] +test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more-itertools", "pytest (>=6,!=8.1.*)", "pytest-ignore-flaky"] +type = ["pytest-mypy"] [metadata] lock-version = "2.0" From 3e45407131b7b1a047bdec6dfb335653fc375434 Mon Sep 17 00:00:00 2001 From: Satya Ortiz-Gagne Date: Wed, 11 Sep 2024 01:11:04 -0400 Subject: [PATCH 3/8] Fix llama3 generation --- .github/workflows/cloud-ci.yml | 8 ++++---- benchmarks/llm/configs/llama3_70B_full.yaml | 2 +- benchmarks/llm/prepare.py | 14 +++++++++++--- config/base.yaml | 7 +++++++ 4 files changed, 23 insertions(+), 8 deletions(-) diff --git a/.github/workflows/cloud-ci.yml b/.github/workflows/cloud-ci.yml index 6db1697c7..e58760a08 100644 --- a/.github/workflows/cloud-ci.yml +++ b/.github/workflows/cloud-ci.yml @@ -115,10 +115,10 @@ jobs: RUN_ON="azure__a100" EXCLUDE="$EXCLUDE,$_MULTI_GPUS,$_MULTI_NODES" ;; - # "2g") - # RUN_ON="azure__a100_x2" - # SELECT="$SELECT,$_MULTI_GPUS" - # ;; + "2g") + RUN_ON="azure__a100_x2" + SELECT="$SELECT,$_MULTI_GPUS" + ;; "4g") RUN_ON="azure__a100_x4" SELECT="$SELECT,$_MULTI_GPUS" diff --git a/benchmarks/llm/configs/llama3_70B_full.yaml b/benchmarks/llm/configs/llama3_70B_full.yaml index ae5cf2afb..5833044d9 100644 --- a/benchmarks/llm/configs/llama3_70B_full.yaml +++ b/benchmarks/llm/configs/llama3_70B_full.yaml @@ -36,7 +36,7 @@ checkpointer: _component_: torchtune.utils.FullModelHFCheckpointer checkpoint_dir: /tmp/Meta-Llama-3.1-70B-Instruct/ checkpoint_files: [ - model-00001-of-00030.safetensors, + model-00001-of-00030.safetensors, model-00002-of-00030.safetensors, model-00003-of-00030.safetensors, model-00004-of-00030.safetensors, diff --git a/benchmarks/llm/prepare.py b/benchmarks/llm/prepare.py index 5f74d2a4e..3ae749ed9 100755 --- a/benchmarks/llm/prepare.py +++ b/benchmarks/llm/prepare.py @@ -23,7 +23,6 @@ class Arguments: recipe: str config: str = None - no_pretrained: bool = False @dataclass @@ -100,12 +99,19 @@ def load_model(recipe, cfg): def generate_weights(args, config): + is_done:Path = args.output_dir / "generated" + if is_done.exists(): + print(f"{args.output_dir}/['*.safetensors'] or ['*consolidated.*.pth'] already generated") + return + if config.get("safetensors", False): params_path = args.output_dir / "config.json" model = LlamaForCausalLM(LlamaConfig(**json.loads(params_path.read_text()))) # Avoid saving this as part of the config. del model.config._name_or_path - model.config.torch_dtype = torch.float16 + # Even if model if loaded with a config.torch_dtype == bf16, model.dtype + # seams to be f32. Force model.dtype to be bf16 + model.to(model.config.torch_dtype) model.save_pretrained(str(args.output_dir), safe_serialization=True) else: @@ -138,6 +144,8 @@ def generate_weights(args, config): conn.send(True) p.join() + is_done.touch() + def main(): parser = ArgumentParser() @@ -154,7 +162,7 @@ def main(): # huggingface_format = config.get("safetensors", False) - pretrained = not args.no_pretrained + pretrained = not config.get("no_pretrained", False) if not pretrained: # if we will generate the weights do not download anyweights diff --git a/config/base.yaml b/config/base.yaml index 14836bda8..f15333458 100644 --- a/config/base.yaml +++ b/config/base.yaml @@ -534,6 +534,7 @@ llm-lora-single: inherits: _llm plan: method: per_gpu + argv: "{milabench_code}/recipes/lora_finetune_single_device.py": true --config: "{milabench_code}/configs/llama3_8B_lora_single_device.yaml" @@ -546,6 +547,7 @@ llm-lora-single: repo_id="meta-llama/Meta-Llama-3.1-8B": true batch_size=8: true gradient_accumulation_steps=8: true + no_pretrained=True: true llm-lora-ddp-gpus: inherits: _llm @@ -565,6 +567,7 @@ llm-lora-ddp-gpus: repo_id="meta-llama/Meta-Llama-3.1-8B": true batch_size=8: true gradient_accumulation_steps=8: true + no_pretrained=True: true llm-lora-ddp-nodes: tags: @@ -587,6 +590,7 @@ llm-lora-ddp-nodes: repo_id="meta-llama/Meta-Llama-3.1-8B": true batch_size=8: true gradient_accumulation_steps=8: true + no_pretrained=True: true num_machines: 2 requires_capabilities: @@ -611,6 +615,7 @@ llm-lora-mp-gpus: repo_id="meta-llama/Meta-Llama-3.1-70B": true batch_size=8: true gradient_accumulation_steps=1: true + no_pretrained=True: true llm-full-mp-gpus: inherits: _llm @@ -631,6 +636,7 @@ llm-full-mp-gpus: safetensors=true: true batch_size=2: true gradient_accumulation_steps=1: true + no_pretrained=True: true llm-full-mp-nodes: tags: @@ -654,6 +660,7 @@ llm-full-mp-nodes: safetensors=true: true batch_size=2: true gradient_accumulation_steps=1: true + no_pretrained=True: true num_machines: 2 requires_capabilities: From 3b207f846416b442cbca9e5346d59208206b53d6 Mon Sep 17 00:00:00 2001 From: Satya Ortiz-Gagne Date: Tue, 17 Sep 2024 21:42:25 -0400 Subject: [PATCH 4/8] Add slurm system setup --- config/base.yaml | 7 +- config/cloud-multinodes-system.yaml | 13 + config/cloud-system.yaml | 24 + config/examples/cloud-multinodes-system.yaml | 10 + config/examples/cloud-system.yaml | 11 + docs/usage.rst | 89 +++- milabench/cli/cloud.py | 51 +- milabench/commands/__init__.py | 11 +- milabench/scripts/covalent/__main__.py | 463 ++++++++++++++---- milabench/scripts/covalent/covalent_bashrc.sh | 46 ++ .../scripts/covalent/milabench_bashrc.sh | 31 ++ .../scripts/covalent/python3/__main__.py | 20 + milabench/scripts/covalent/requirements.txt | 3 +- milabench/scripts/utils.py | 15 +- 14 files changed, 669 insertions(+), 125 deletions(-) create mode 100644 milabench/scripts/covalent/covalent_bashrc.sh create mode 100644 milabench/scripts/covalent/milabench_bashrc.sh create mode 100644 milabench/scripts/covalent/python3/__main__.py diff --git a/config/base.yaml b/config/base.yaml index f15333458..2509a5132 100644 --- a/config/base.yaml +++ b/config/base.yaml @@ -28,6 +28,7 @@ _torchvision: --loader: pytorch --data: "{milabench_data}/FakeImageNet" + _torchvision_ddp: inherits: _defaults definition: ../benchmarks/torchvision_ddp @@ -113,6 +114,7 @@ _timm: --dataset: "FakeImageNet" --workers: "auto({n_worker}, 8)" + _accelerate_opt: inherits: _defaults tags: @@ -149,6 +151,7 @@ _accelerate_opt: use_deepspeed: true num_machines: 1 + fp16: inherits: _flops @@ -388,6 +391,7 @@ brax: --num-minibatches: 32 --num-envs: 8192 + _diffusion: inherits: _defaults definition: ../benchmarks/diffusion @@ -530,11 +534,11 @@ _llm: definition: ../benchmarks/llm install_group: torch + llm-lora-single: inherits: _llm plan: method: per_gpu - argv: "{milabench_code}/recipes/lora_finetune_single_device.py": true --config: "{milabench_code}/configs/llama3_8B_lora_single_device.yaml" @@ -596,6 +600,7 @@ llm-lora-ddp-nodes: requires_capabilities: - "len(nodes) >= ${num_machines}" + llm-lora-mp-gpus: inherits: _llm plan: diff --git a/config/cloud-multinodes-system.yaml b/config/cloud-multinodes-system.yaml index 4f7fae391..96cf7c11d 100644 --- a/config/cloud-multinodes-system.yaml +++ b/config/cloud-multinodes-system.yaml @@ -38,3 +38,16 @@ system: size: Standard_NV72ads_A10_v5 location: eastus2 disk_size: 512 + slurm__a100_x2: + address: localhost + bashrc_path: "{bashrc_path}" + remote_workdir: "scratch/cov-{job_uuid}-workdir" + use_srun: null + options: + ntasks-per-node: 1 + gpus-per-task: a100l:2 + cpus-per-task: 12 + time: "3:0:0" + mem: 64000 + partition: short-unkillable + nodelist: cn-g[001-029] diff --git a/config/cloud-system.yaml b/config/cloud-system.yaml index 056ef3640..2bee1ef13 100644 --- a/config/cloud-system.yaml +++ b/config/cloud-system.yaml @@ -38,3 +38,27 @@ system: size: Standard_NV72ads_A10_v5 location: eastus2 disk_size: 512 + slurm__a100_x1: + address: localhost + bashrc_path: "{bashrc_path}" + remote_workdir: "scratch/cov-{job_uuid}-workdir" + use_srun: null + options: + ntasks-per-node: 1 + gpus-per-task: a100l:1 + cpus-per-task: 6 + time: "3:0:0" + mem: 32000 + partition: unkillable + slurm__a100_x4: + address: localhost + bashrc_path: "{bashrc_path}" + remote_workdir: "scratch/cov-{job_uuid}-workdir" + use_srun: null + options: + ntasks-per-node: 1 + gpus-per-task: a100l:4 + cpus-per-task: 24 + time: "3:0:0" + mem: 128000 + partition: short-unkillable diff --git a/config/examples/cloud-multinodes-system.yaml b/config/examples/cloud-multinodes-system.yaml index 5066af5eb..27f1e41b1 100644 --- a/config/examples/cloud-multinodes-system.yaml +++ b/config/examples/cloud-multinodes-system.yaml @@ -35,3 +35,13 @@ system: volume_size: 8 region: us-east-2 state_id: 71669879043a3864225aabb94f91a2d4 + slurm: + address: localhost + bashrc_path: "{bashrc_path}" + remote_workdir: "scratch/cov-{job_uuid}-workdir" + use_srun: null + options: + ntasks-per-node: 1 + cpus-per-task: 1 + time: "0:30:0" + mem: 1000 diff --git a/config/examples/cloud-system.yaml b/config/examples/cloud-system.yaml index b3d1f70aa..d3d3942cc 100644 --- a/config/examples/cloud-system.yaml +++ b/config/examples/cloud-system.yaml @@ -28,3 +28,14 @@ system: instance_type: t2.micro volume_size: 8 region: us-east-2 + slurm: + # covalent-slurm-plugin args + address: localhost + bashrc_path: "{bashrc_path}" + remote_workdir: "scratch/cov-{job_uuid}-workdir" + use_srun: null + options: + ntasks-per-node: 1 + cpus-per-task: 1 + time: "0:30:0" + mem: 1000 diff --git a/docs/usage.rst b/docs/usage.rst index b2a25d85d..4f0fa6f13 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -102,7 +102,7 @@ Create a cloud system configuration ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Add a ``cloud_profiles`` section to the ``system`` configuration which lists the -supported cloud profiles. +supported cloud and slurm profiles. .. notes:: @@ -150,14 +150,95 @@ Run milabench on the cloud ^^^^^^^^^^^^^^^^^^^^^^^^^^ 1. | Initialize the cloud instances - | ``milabench cloud --system {{SYSTEM_CONFIG.YAML}} --setup --run-on {{PROFILE}} >{{SYSTEM_CLOUD_CONFIG.YAML}}`` + | ``milabench cloud --setup --system {{SYSTEM_CONFIG.YAML}} --run-on {{PROFILE}} >{{SYSTEM_CLOUD_CONFIG.YAML}}`` 2. | Prepare, install and run milabench | ``milabench [prepare|install|run] --system {{SYSTEM_CLOUD_CONFIG.YAML}}`` 3. | Destroy the cloud instances - | ``milabench teardown --system {{SYSTEM_CLOUD_CONFIG.YAML}} --run-on {{PROFILE}}`` + | ``milabench cloud --teardown --system {{SYSTEM_CLOUD_CONFIG.YAML}} --run-on {{PROFILE}}`` | or - | ``milabench teardown --system {{SYSTEM_CLOUD_CONFIG.YAML}} --run-on {{PLATFORM}} --all`` + | ``milabench cloud --teardown --system {{SYSTEM_CLOUD_CONFIG.YAML}} --run-on {{PLATFORM}} --all`` | to destroy not just a single cloud instance but all instances on a specified platform that were instanced from the current local machine + + +Use milabench on slurm +~~~~~~~~~~~~~~~~~~~~~~ + + +Create a slurm system configuration +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Add a ``cloud_profiles`` section to the ``system`` configuration which lists the +supported cloud and slurm profiles. + +.. notes:: + + Nodes that should be created on the cloud should have the ``1.1.1.1`` ip + address placeholder. Other ip addresses will be used as-is and no cloud + instance will be created for that node + +.. notes:: + + A cloud profile entry needs to start with a covalent plugin (e.g. `slurm`). To + define multiple profiles on the same cloud platform, use the form + ``{PLATFORM}__{PROFILE_NAME}`` (e.g. ``slurm__profile``). All cloud profile + attributes will be used as is as argument for the target covalent plugin + +.. code-block:: yaml + + system: + nodes: + - name: manager + # Use 1.1.1.1 as an ip placeholder + ip: 1.1.1.1 + main: true + user: + - name: node1 + ip: 1.1.1.1 + main: false + user: + + # Cloud instances profiles + cloud_profiles: + # The cloud platform to use in the form of {PLATFORM} or + # {PLATFORM}__{PROFILE_NAME} + slurm: + username: usename + address: localhost + ssh_key_file: ssh_key_file + # bashrc_path will be replaced by the content of + # milabench/scripts/covalent/covalent_bashrc.sh + bashrc_path: "{bashrc_path}" + # job_uuid will be replaced by the generated job's uuid + remote_workdir: "cov-{job_uuid}-workdir" + use_srun: null + options: + ntasks-per-node: 1 + cpus-per-task: 1 + time: "0:30:0" + mem: 1000 + + +Run milabench on slurm +^^^^^^^^^^^^^^^^^^^^^^ + +1. | Initialize the slurm instances + | ``milabench cloud --setup --system {{SYSTEM_CONFIG.YAML}} --run-on {{PROFILE}} >{{SYSTEM_SLURM_CONFIG.YAML}}`` + +2. | Prepare, install and run milabench + | ``milabench [prepare|install|run] --system {{SYSTEM_SLURM_CONFIG.YAML}}`` + +3. | Destroy the slurm instances + | ``milabench cloud --teardown --system {{SYSTEM_SLURM_CONFIG.YAML}} --run-on {{PROFILE}}`` + +.. notes:: + + Because the milabench's path is expected to be the same on local machine and + the remote machine, it's currently necessary to run the commands from the + slurm cluster. As the ``milabench cloud --[setup|teardown]`` commands requires + a covalent server to run and to avoid overloading the login nodes resources, + it's preferable to request a cpu compute node which will host to the covalent + server. An allocation with minimal resources like ``--nodes 1 --cpus-per-task + 1 --mem 2000`` should be enough. diff --git a/milabench/cli/cloud.py b/milabench/cli/cloud.py index 859cdab87..1d89b69e4 100644 --- a/milabench/cli/cloud.py +++ b/milabench/cli/cloud.py @@ -60,29 +60,33 @@ def manage_cloud(pack, run_on, action="setup"): "private_ip":(lambda v: ("internal_ip",v)), "username":(lambda v: ("user",v)), "ssh_key_file":(lambda v: ("key",v)), - # "env":(lambda v: ("env",[".", v, ";", "conda", "activate", "milabench", "&&"])), + "env":(lambda v: ("env",[".", v, "milabench", "&&"])), + "slurm_job_id":(lambda v: ("slurm_job_id",v)), } - plan_params = deepcopy(pack.config["system"]["cloud_profiles"][run_on]) + plan_params = pack.config["system"]["cloud_profiles"][run_on] run_on, *profile = run_on.split("__") profile = profile[0] if profile else "" default_state_prefix = profile or run_on default_state_id = "_".join((pack.config["hash"][:6], blabla())) - local_base = pack.dirs.base.absolute() - local_data_dir = _get_common_dir(ROOT_FOLDER.parent, local_base.parent) - if local_data_dir is None: - local_data_dir = local_base.parent - remote_data_dir = XPath("/data") / local_data_dir.name + plan_params["state_prefix"] = plan_params.get("state_prefix", default_state_prefix) + plan_params["state_id"] = plan_params.get("state_id", default_state_id) + plan_params["keep_alive"] = None + + # local_base = pack.dirs.base.absolute() + # local_data_dir = _get_common_dir(ROOT_FOLDER.parent, local_base.parent) + # if local_data_dir is None: + # local_data_dir = local_base.parent + # remote_data_dir = XPath("/data") / local_data_dir.name + + plan_params_copy = deepcopy(plan_params) nodes = iter(enumerate(pack.config["system"]["nodes"])) for i, n in nodes: - if n["ip"] != "1.1.1.1": + if n["ip"] != "1.1.1.1" and action == _SETUP: continue - plan_params["state_prefix"] = plan_params.get("state_prefix", default_state_prefix) - plan_params["state_id"] = plan_params.get("state_id", default_state_id) - plan_params["cluster_size"] = max(len(pack.config["system"]["nodes"]), i + 1) - plan_params["keep_alive"] = None + plan_params_copy["cluster_size"] = max(len(pack.config["system"]["nodes"]), i + 1) import milabench.scripts.covalent as cv @@ -101,17 +105,17 @@ def manage_cloud(pack, run_on, action="setup"): "-m", cv.__name__, run_on, f"--{action}", - *_flatten_cli_args(**plan_params) + *_flatten_cli_args(**plan_params_copy) ] - if action == _SETUP: - cmd += [ - "--", - "bash", "-c", - _or_sudo(f"mkdir -p '{local_data_dir.parent}'") + - " && " + _or_sudo(f"chmod a+rwX '{local_data_dir.parent}'") + - f" && mkdir -p '{remote_data_dir}'" - f" && ln -sfT '{remote_data_dir}' '{local_data_dir}'" - ] + # if action == _SETUP: + # cmd += [ + # "--", + # "bash", "-c", + # _or_sudo(f"mkdir -p '{local_data_dir.parent}'") + + # " && " + _or_sudo(f"chmod a+rwX '{local_data_dir.parent}'") + + # f" && mkdir -p '{remote_data_dir}'" + # f" && ln -sfT '{remote_data_dir}' '{local_data_dir}'" + # ] p = subprocess.Popen( cmd, stdout=subprocess.PIPE, @@ -155,6 +159,9 @@ def manage_cloud(pack, run_on, action="setup"): stderr ) + if action == _TEARDOWN: + break + return pack.config["system"] diff --git a/milabench/commands/__init__.py b/milabench/commands/__init__.py index ad281b23b..fcd6ccbf4 100644 --- a/milabench/commands/__init__.py +++ b/milabench/commands/__init__.py @@ -450,6 +450,11 @@ def _find_node_config(self) -> Dict: return n return {} + def _load_env(self, node): + if node.get("env", None): + return node["env"] + return [] + def is_local(self): localnode = self.pack.config["system"]["self"] @@ -484,7 +489,7 @@ def _argv(self, **kwargs) -> List: argv.append(f"-p{self.port}") argv.append(host) - return argv # + ["env", "-i"] + return argv + self._load_env(node) class SCPCommand(SSHCommand, CmdCommand): @@ -505,6 +510,10 @@ def __init__( self.src = src self.dest = dest if dest is not None else self.src + def _load_env(self, node): + del node + return [] + def _argv(self, **kwargs) -> List: argv = super()._argv(**kwargs) diff --git a/milabench/scripts/covalent/__main__.py b/milabench/scripts/covalent/__main__.py index d4de9d932..689eda8fa 100644 --- a/milabench/scripts/covalent/__main__.py +++ b/milabench/scripts/covalent/__main__.py @@ -1,9 +1,50 @@ import argparse +import ast import os import pathlib import subprocess import sys import tempfile +from time import sleep +import uuid + + +def _arg_pop(args:argparse.Namespace, key:str): + value = args.__getattribute__(key) + args.__delattr__(key) + return value + +ARGS_DEFAULT_SETUP = { + "slurm": { + "state_prefix": {}, + "state_id": {}, + "cluster_size": {}, + "keep_alive": {"action": "store_true"}, + } +} + +ARGS_MAP = { + "slurm": { + "state_prefix": lambda args, k:_arg_pop(args, k), + "state_id": lambda args, k:args.options.setdefault("job-name", _arg_pop(args, k)), + "cluster_size": lambda args, k:args.options.setdefault("nodes", _arg_pop(args, k)), + "keep_alive": lambda args, k:_arg_pop(args, k), + } +} + +_SETUP = {} + +_TEARDOWN = {} + +_CONNECTION_ATTRIBUTES = { + "hostname": None, + "username": None, + "ssh_key_file": None, + "private_ip": None, + "env": None, + "python_path": None, + "slurm_job_id": None +} def serve(*argv): @@ -19,69 +60,328 @@ def _get_executor_kwargs(args): } -def executor(executor_cls, args, *argv): +def _wait_for_any(*dispatch_ids): import covalent as ct - def _popen(cmd, *args, _env=None, **kwargs): - _env = _env if _env is not None else {} + dispatch_ids = set(dispatch_ids) + while True: + for dispatch_id in set(dispatch_ids): + status = ct.get_result( + dispatch_id=dispatch_id, + wait=False, + status_only=True + )["status"] + if status in [ct.status.COMPLETED]: + yield dispatch_id + dispatch_ids.remove(dispatch_id) + elif status in [ct.status.FAILED, ct.status.CANCELLED]: + raise RuntimeError(f"Job {dispatch_id} failed") + sleep(5) + + +def _format(lines:list, **template_kv): + for l in lines: + for k, v in template_kv.items(): + if "{{" + k + "}}" in l: + yield l[:l.find("{{")] + v + l[l.find("}}")+2:] + break + else: + yield l - for envvar in _env.keys(): - envvar_val = _env[envvar] - if not envvar_val: - continue +def _popen(cmd, *args, _env=None, **kwargs): + _env = _env if _env is not None else {} - envvar_val = pathlib.Path(envvar_val).expanduser() - if str(envvar_val) != _env[envvar]: - _env[envvar] = str(envvar_val) + for envvar in _env.keys(): + envvar_val = _env[envvar] - if "MILABENCH_CONFIG_CONTENT" in _env: - _config_dir = pathlib.Path(_env["MILABENCH_CONFIG"]).parent - with tempfile.NamedTemporaryFile("wt", dir=str(_config_dir), suffix=".yaml", delete=False) as _f: - _f.write(_env["MILABENCH_CONFIG_CONTENT"]) - _env["MILABENCH_CONFIG"] = _f.name + if not envvar_val: + continue - try: - cmd = (str(pathlib.Path(cmd[0]).expanduser()), *cmd[1:]) - except IndexError: - pass - - cwd = kwargs.pop("cwd", None) - if cwd is not None: - cwd = str(pathlib.Path(cwd).expanduser()) - kwargs["cwd"] = cwd - - _env = {**os.environ.copy(), **kwargs.pop("env", {}), **_env} - - kwargs = { - **kwargs, - "env": _env, - "stdout": subprocess.PIPE, - "stderr": subprocess.PIPE, - } - p = subprocess.Popen(cmd, *args, **kwargs) + envvar_val = pathlib.Path(envvar_val).expanduser() + if str(envvar_val) != _env[envvar]: + _env[envvar] = str(envvar_val) + + if "MILABENCH_CONFIG_CONTENT" in _env: + _config_dir = pathlib.Path(_env["MILABENCH_CONFIG"]).parent + with tempfile.NamedTemporaryFile("wt", dir=str(_config_dir), suffix=".yaml", delete=False) as _f: + _f.write(_env["MILABENCH_CONFIG_CONTENT"]) + _env["MILABENCH_CONFIG"] = _f.name + + try: + cmd = (str(pathlib.Path(cmd[0]).expanduser()), *cmd[1:]) + except IndexError: + pass + + cwd = kwargs.pop("cwd", None) + if cwd is not None: + cwd = str(pathlib.Path(cwd).expanduser()) + kwargs["cwd"] = cwd + + _env = {**os.environ.copy(), **kwargs.pop("env", {}), **_env} + + kwargs = { + **kwargs, + "env": _env, + "stdout": subprocess.PIPE, + "stderr": subprocess.PIPE, + } + p = subprocess.Popen(cmd, *args, **kwargs) + + stdout_chunks = [] + while True: + line = p.stdout.readline() + if not line: + break + line_str = line.decode("utf-8").strip() + stdout_chunks.append(line_str) + print(line_str) + + _, stderr = p.communicate() + stderr = stderr.decode("utf-8").strip() + stdout = os.linesep.join(stdout_chunks) + + if p.returncode != 0: + raise subprocess.CalledProcessError( + p.returncode, + (cmd, args, kwargs), + stdout, + stderr + ) + return p.returncode, stdout, stderr + + +def _setup_terraform(executor:"ct.executor.BaseExecutor"): + import covalent as ct + + result = ct.dispatch_sync( + ct.lattice(executor.get_connection_attributes) + )().result + + assert result and result[0] + + all_connection_attributes, _ = result + master_host:str = next(iter(all_connection_attributes)) + + if len(all_connection_attributes) > 1: + # Add master node to known host to avoid unknown host error The + # authenticity of host '[hostname] ([IP address])' can't be established. + new_host = subprocess.run( + ["ssh-keyscan", master_host], + stdout=subprocess.PIPE, + check=True + ).stdout.decode("utf8") + known_hosts = pathlib.Path("~/.ssh/known_hosts").expanduser() + with known_hosts.open("at") as _f: + _f.write(new_host) + + # Add ssh file to master node to allow connections to worker nodes + ssh_key_file = all_connection_attributes[master_host]["ssh_key_file"] + fn = pathlib.Path(ssh_key_file) + result = ct.dispatch_sync( + ct.lattice(executor.cp_to_remote) + )(f".ssh/{fn.name.split('.')[0]}", str(fn)) + + assert result.status == ct.status.COMPLETED + + return all_connection_attributes + + +def _teardown_terraform(executor:"ct.executor.BaseExecutor"): + result = executor.stop_cloud_instance().result + assert result is not None + + +def _slurm_executor(executor:"ct.executor.SlurmExecutor", job_uuid:uuid.UUID): + import covalent as ct + + _executor = ct.executor.SlurmExecutor() + _executor.from_dict(executor.to_dict()) + + executor = _executor + executor.conda_env = executor.conda_env or "covalent" + bashrc_path = f"""'' +{pathlib.Path(__file__).with_name("covalent_bashrc.sh").read_text()} +""" + bashrc_path = "\n".join( + _format( + bashrc_path.splitlines(), + conda_env=executor.conda_env, + python_version=f"{sys.version_info.major}.{sys.version_info.minor}", + covalent_version=ct.__version__, + ) + ) + executor.bashrc_path = executor.bashrc_path or "{bashrc_path}" + if "{bashrc_path}" in executor.bashrc_path: + executor.bashrc_path = executor.bashrc_path.format(bashrc_path=bashrc_path) + executor.remote_workdir = executor.remote_workdir or "cov-{job_uuid}-workdir" + executor.remote_workdir = executor.remote_workdir.format(job_uuid=job_uuid) + executor.options["job-name"] = ( + executor.options.get("job-name", None) or f"cov-{job_uuid}" + ) + + return executor - stdout_chunks = [] + +def _setup_slurm(executor:"ct.executor.SlurmExecutor"): + import covalent as ct + + job_uuid = uuid.uuid4() + job_file = f"covalent_job_{job_uuid}" + + _executor = ct.executor.SlurmExecutor() + _executor.from_dict(executor.to_dict()) + + executor = _slurm_executor(executor, job_uuid) + + job_connection_executor = ct.executor.SlurmExecutor() + job_connection_executor.from_dict(executor.to_dict()) + # Store job connection attributes + job_connection_executor.prerun_commands = f""" +# print connection attributes +printenv | + grep -E ".*SLURM.*NODENAME|.*SLURM.*JOB_ID" | + sort -u >>"{job_file}" && +srun printenv | + grep -E ".*SLURM.*NODENAME|.*SLURM.*JOB_ID" | + sort -u >>"{job_file}" && +echo "USERNAME=$USER" >>"{job_file}" && +echo "{job_uuid}" >>"{job_file}" +""".splitlines() + + query_executor = ct.executor.SlurmExecutor() + query_executor.from_dict(executor.to_dict()) + query_executor.options = { + "nodes": 1, + "cpus-per-task": 1, + "mem": 1000, + "job-name": executor.options["job-name"], + } + + @ct.electron() + def _empty(): + pass + + @ct.electron() + def _keep_alive(): while True: - line = p.stdout.readline() - if not line: - break - line_str = line.decode("utf-8").strip() - stdout_chunks.append(line_str) - print(line_str) - - _, stderr = p.communicate() - stderr = stderr.decode("utf-8").strip() - stdout = os.linesep.join(stdout_chunks) - - if p.returncode != 0: - raise subprocess.CalledProcessError( - p.returncode, - (cmd, args, kwargs), - stdout, - stderr + sleep(60) + + @ct.electron() + def _query_connection_attributes(milabench_bashrc:str=""): + _job_file = pathlib.Path(job_file).expanduser() + _job_file.touch() + content = _job_file.read_text().splitlines() + while (not content or content[-1].strip() != f"{job_uuid}"): + sleep(5) + content = _job_file.read_text().splitlines() + + nodes = [] + connection_attributes = _CONNECTION_ATTRIBUTES.copy() + + milabench_bashrc = "\n".join( + _format( + milabench_bashrc.splitlines(), + milabench_env="cov-slurm-milabench", + python_version=f"{sys.version_info.major}.{sys.version_info.minor}", ) - return p.returncode, stdout, stderr + ) + if milabench_bashrc: + milabench_bashrc_file = _job_file.with_name("milabench_bashrc.sh").resolve() + milabench_bashrc_file.write_text(milabench_bashrc) + connection_attributes["env"] = str(milabench_bashrc_file) + + for l in _job_file.read_text().splitlines(): + try: + key, value = l.strip().split("=") + except ValueError: + # end flag + break + if "NODENAME" in key and value not in nodes: + nodes.append(value) + elif "USERNAME" in key: + connection_attributes["username"] = value + elif "JOB_ID" in key: + connection_attributes["slurm_job_id"] = value + + return { + hostname: { + **connection_attributes, + **{ + "hostname": hostname, + "private_ip": hostname, + }, + } + for hostname in nodes + } + + try: + # setup covalent for jobs + next(_wait_for_any(ct.dispatch(ct.lattice(_empty, executor=query_executor))())) + # setup nodes and retrieve connection attributes + job_dispatch_id = ct.dispatch( + ct.lattice( + lambda:_keep_alive(), + executor=job_connection_executor + ), + disable_run=False + )() + query_dispatch_id = ct.dispatch( + ct.lattice( + _query_connection_attributes, + executor=query_executor + ), + disable_run=False + )( + milabench_bashrc=pathlib.Path(__file__).with_name("milabench_bashrc.sh").read_text() + ) + next(_wait_for_any(job_dispatch_id, query_dispatch_id)) + all_connection_attributes = ct.get_result( + dispatch_id=query_dispatch_id, + wait=False + ).result + + assert all_connection_attributes + + except: + _teardown_slurm(query_executor) + raise + + return all_connection_attributes + + +def _teardown_slurm(executor:"ct.executor.SlurmExecutor"): + import covalent as ct + + @ct.electron() + def _empty(): + pass + + assert executor.options["job-name"], "Jobs to teardown must have an explicit name" + + _exec = _slurm_executor(executor, "DELETE") + _exec.options = { + "nodes": 1, + "cpus-per-task": 1, + "mem": 1000, + "job-name": executor.options["job-name"], + } + _exec.prerun_commands = f""" +# cancel jobs +scancel --jobname="{_exec.options['job-name']}" +""".splitlines() + ct.dispatch_sync(ct.lattice(_empty, executor=_exec))() + + +def executor(executor_cls, args:argparse.Namespace, *argv): + import covalent as ct + + _SETUP[ct.executor.AzureExecutor] = _setup_terraform + _SETUP[ct.executor.EC2Executor] = _setup_terraform + _SETUP[ct.executor.SlurmExecutor] = _setup_slurm + _TEARDOWN[ct.executor.AzureExecutor] = _teardown_terraform + _TEARDOWN[ct.executor.EC2Executor] = _teardown_terraform + _TEARDOWN[ct.executor.SlurmExecutor] = _teardown_slurm executor:ct.executor.BaseExecutor = executor_cls( **_get_executor_kwargs(args), @@ -89,45 +389,13 @@ def _popen(cmd, *args, _env=None, **kwargs): return_code = 0 try: if args.setup: - result = ct.dispatch_sync( - ct.lattice(executor.get_connection_attributes) - )().result - - assert result and result[0] - - all_connection_attributes, _ = result - master_host:str = None - for hostname, connection_attributes in all_connection_attributes.items(): + for hostname, connection_attributes in _SETUP[executor_cls](executor).items(): print(f"hostname::>{hostname}") for attribute,value in connection_attributes.items(): - if attribute == "hostname": + if attribute == "hostname" or value is None: continue print(f"{attribute}::>{value}") - master_host = master_host or hostname - - if len(all_connection_attributes) > 1: - # Add master node to known host to avoid unknown host error - # The authenticity of host '[hostname] ([IP address])' can't be established. - new_host = subprocess.run( - ["ssh-keyscan", master_host], - stdout=subprocess.PIPE, - check=True - ).stdout.decode("utf8") - known_hosts = pathlib.Path("~/.ssh/known_hosts").expanduser() - with known_hosts.open("at") as _f: - _f.write(new_host) - - # Add ssh file to master node to allow connections to worker - # nodes - ssh_key_file = all_connection_attributes[master_host]["ssh_key_file"] - fn = pathlib.Path(ssh_key_file) - result = ct.dispatch_sync( - ct.lattice(executor.cp_to_remote) - )(f".ssh/{fn.name.split('.')[0]}", str(fn)) - - assert result.status == ct.status.COMPLETED - if argv: result = ct.dispatch_sync( ct.lattice(executor.list_running_instances) @@ -166,8 +434,7 @@ def _popen(cmd, *args, _env=None, **kwargs): finally: if args.teardown: - result = executor.stop_cloud_instance().result - assert result is not None + _TEARDOWN[executor_cls](executor) return return_code @@ -187,7 +454,7 @@ def main(argv=None): subparsers = parser.add_subparsers() subparser = subparsers.add_parser("serve") subparser.add_argument(f"argv", nargs=argparse.REMAINDER) - for p in ("azure","ec2"): + for p in ("azure", "ec2", "slurm"): try: config = ct.get_config(f"executors.{p}") except KeyError: @@ -201,10 +468,21 @@ def main(argv=None): add_argument_kwargs = {} if isinstance(default, bool): add_argument_kwargs["action"] = "store_false" if default else "store_true" + elif any(isinstance(default, t) for t in [dict, list]): + add_argument_kwargs["type"] = ast.literal_eval + add_argument_kwargs["default"] = str(default) else: add_argument_kwargs["default"] = default subparser.add_argument(f"--{param.replace('_', '-')}", **add_argument_kwargs) + for param, add_argument_kwargs in ARGS_DEFAULT_SETUP.get(p, {}).items(): + if param in config: + raise ValueError( + f"Found existing argument {param} in both {config} and" + f" {ARGS_DEFAULT_SETUP}" + ) + subparser.add_argument(f"--{param.replace('_', '-')}", **add_argument_kwargs) + try: cv_argv, argv = argv[:argv.index("--")], argv[argv.index("--")+1:] except ValueError: @@ -212,6 +490,9 @@ def main(argv=None): args = parser.parse_args(cv_argv) + for arg, _map in ARGS_MAP.get(cv_argv[0], {}).items(): + _map(args, arg) + if cv_argv[0] == "serve": assert not argv return serve(*args.argv) @@ -219,6 +500,8 @@ def main(argv=None): executor_cls = ct.executor.AzureExecutor elif cv_argv[0] == "ec2": executor_cls = ct.executor.EC2Executor + elif cv_argv[0] == "slurm": + executor_cls = ct.executor.SlurmExecutor else: raise diff --git a/milabench/scripts/covalent/covalent_bashrc.sh b/milabench/scripts/covalent/covalent_bashrc.sh new file mode 100644 index 000000000..197620c1c --- /dev/null +++ b/milabench/scripts/covalent/covalent_bashrc.sh @@ -0,0 +1,46 @@ +function _get_options { + set +o | cut -d' ' -f2- | while read set_option + do + echo "${set_option}" + done +} + +function _set_options { + while [[ $# -gt 0 ]] + do + local _arg="$1"; shift + case "${_arg}" in + -o) set -o "$1"; shift ;; + +o) set +o "$1"; shift ;; + -h | --help | *) + exit 1 + ;; + esac + done +} + +_options=$(_get_options) +set -o errexit -o pipefail + +_NAME="{{conda_env}}" +conda --version >&2 2>/dev/null || module load anaconda/3 +# {{python_version}} needs to be formatted +conda activate ${_NAME} || conda create -y -n ${_NAME} "python={{python_version}}" virtualenv +conda activate ${_NAME} + +function conda { + echo "DEACTIVATED conda" "$@" >&2 +} + +if [ ! -f venv/bin/activate ] +then + python3 -m virtualenv venv/ + . venv/bin/activate + # {{covalent_version}} needs to be formatted + python3 -m pip install "covalent=={{covalent_version}}" +else + . venv/bin/activate +fi + +_set_options $_options +unset _options diff --git a/milabench/scripts/covalent/milabench_bashrc.sh b/milabench/scripts/covalent/milabench_bashrc.sh new file mode 100644 index 000000000..1ee96d788 --- /dev/null +++ b/milabench/scripts/covalent/milabench_bashrc.sh @@ -0,0 +1,31 @@ +function _get_options { + set +o | cut -d' ' -f2- | while read set_option + do + echo "${set_option}" + done +} + +function _set_options { + while [[ $# -gt 0 ]] + do + local _arg="$1"; shift + case "${_arg}" in + -o) set -o "$1"; shift ;; + +o) set +o "$1"; shift ;; + -h | --help | *) + exit 1 + ;; + esac + done +} + +_options=$(_get_options) +set -o errexit -o pipefail + +_NAME="{{milabench_env}}" +conda --version >&2 2>/dev/null || module load anaconda/3 +conda activate ${_NAME} || conda create -y -n ${_NAME} "python={{python_version}}" virtualenv +conda activate ${_NAME} + +_set_options $_options +unset _options diff --git a/milabench/scripts/covalent/python3/__main__.py b/milabench/scripts/covalent/python3/__main__.py new file mode 100644 index 000000000..186d9a170 --- /dev/null +++ b/milabench/scripts/covalent/python3/__main__.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 + +import subprocess +import sys + + +def main(argv=None): + if argv is None: + argv = sys.argv[1:] + + from ...utils import get_module_venv + from .. import __main__ + check_if_module = "import covalent" + python3, env = get_module_venv(__main__.__file__, check_if_module) + + return subprocess.call([python3, *argv], env=env) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/milabench/scripts/covalent/requirements.txt b/milabench/scripts/covalent/requirements.txt index b70efc793..4b322dc3f 100644 --- a/milabench/scripts/covalent/requirements.txt +++ b/milabench/scripts/covalent/requirements.txt @@ -1,3 +1,4 @@ covalent==0.232 covalent-ec2-plugin @ git+https://github.com/satyaog/covalent-ec2-plugin.git@feature/milabench -covalent-azure-plugin @ git+https://github.com/satyaog/covalent-azure-plugin.git@feature/milabench \ No newline at end of file +covalent-azure-plugin @ git+https://github.com/satyaog/covalent-azure-plugin.git@feature/milabench +covalent-slurm-plugin diff --git a/milabench/scripts/utils.py b/milabench/scripts/utils.py index 5aec72d06..022481089 100644 --- a/milabench/scripts/utils.py +++ b/milabench/scripts/utils.py @@ -1,3 +1,4 @@ +import getpass import json import pathlib import subprocess @@ -16,9 +17,9 @@ def get_venv(venv:pathlib.Path) -> dict: return json.loads(env) -def run_in_module_venv(module_main:str, check_if_module:str, argv:list=None): +def get_module_venv(module_main:str, check_if_module:str): module = pathlib.Path(module_main).resolve().parent - cache_dir = pathlib.Path(f"/tmp/milabench/{module.name}_venv") + cache_dir = pathlib.Path(f"/tmp/{getpass.getuser()}/milabench/{module.name}_venv") python3 = str(cache_dir / "bin/python3") try: subprocess.run([python3, "-c", check_if_module], check=True, @@ -38,7 +39,9 @@ def run_in_module_venv(module_main:str, check_if_module:str, argv:list=None): str(module / "requirements.txt") ], stdout=sys.stderr, check=True) subprocess.run([python3, "-c", check_if_module], check=True, stdout=sys.stderr) - return subprocess.call( - [python3, module_main, *argv], - env=get_venv(cache_dir) - ) \ No newline at end of file + return python3, get_venv(cache_dir) + + +def run_in_module_venv(module_main:str, check_if_module:str, argv:list=None): + python3, env = get_module_venv(module_main, check_if_module) + return subprocess.call([python3, module_main, *argv], env=env) \ No newline at end of file From 8cc4926cb822f2b4d7871753a5dcf014e89c5395 Mon Sep 17 00:00:00 2001 From: Satya Ortiz-Gagne Date: Mon, 23 Sep 2024 13:52:36 -0400 Subject: [PATCH 5/8] Fix llama3 generation --- benchmarks/llm/prepare.py | 12 +++++++----- config/base.yaml | 12 ++++++------ 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/benchmarks/llm/prepare.py b/benchmarks/llm/prepare.py index 3ae749ed9..96bd99638 100755 --- a/benchmarks/llm/prepare.py +++ b/benchmarks/llm/prepare.py @@ -68,6 +68,7 @@ def generate_model( params = json.loads(params_path.read_text()) model = llama.model.Transformer(ModelArgs(**params)) + model.to(torch.bfloat16) torch.save(model.state_dict(), params_path.with_name(f"consolidated.{rank:02}.pth")) except Exception as e: @@ -117,11 +118,12 @@ def generate_weights(args, config): else: # Note that at the time of writing torchtune doesn't support multi-*.pth # files loading + ctx = multiprocessing.get_context("spawn") params_path = next(args.output_dir.glob("**/params.json")) model_parallel_size = len(config["checkpointer"]["checkpoint_files"]) - pipes = [multiprocessing.Pipe() for _ in range(model_parallel_size)] + pipes = [ctx.Pipe() for _ in range(model_parallel_size)] processes = [ - multiprocessing.Process( + ctx.Process( target=generate_model, args=[conn, params_path, rank, model_parallel_size] ) @@ -162,9 +164,9 @@ def main(): # huggingface_format = config.get("safetensors", False) - pretrained = not config.get("no_pretrained", False) + untrained = config.get("untrained", False) - if not pretrained: + if untrained: # if we will generate the weights do not download anyweights ignore_patterns = ["*.safetensors", "*consolidated.*.pth"] @@ -203,7 +205,7 @@ def main(): args = parser.parse_args(download_args) parser.run(args) - if not pretrained: + if untrained: generate_weights(args, config) if "qlora" in config.get("model", {}).get("_component_", ""): diff --git a/config/base.yaml b/config/base.yaml index 1ee4b182c..5ddae2a54 100644 --- a/config/base.yaml +++ b/config/base.yaml @@ -550,7 +550,7 @@ llm-lora-single: repo_id="meta-llama/Meta-Llama-3.1-8B": true batch_size=8: true gradient_accumulation_steps=8: true - no_pretrained=True: true + untrained=True: true llm-lora-ddp-gpus: @@ -571,7 +571,7 @@ llm-lora-ddp-gpus: repo_id="meta-llama/Meta-Llama-3.1-8B": true batch_size=8: true gradient_accumulation_steps=8: true - no_pretrained=True: true + untrained=True: true llm-lora-ddp-nodes: @@ -595,7 +595,7 @@ llm-lora-ddp-nodes: repo_id="meta-llama/Meta-Llama-3.1-8B": true batch_size=8: true gradient_accumulation_steps=8: true - no_pretrained=True: true + untrained=True: true num_machines: 2 requires_capabilities: @@ -621,7 +621,7 @@ llm-lora-mp-gpus: repo_id="meta-llama/Meta-Llama-3.1-70B": true batch_size=8: true gradient_accumulation_steps=1: true - no_pretrained=True: true + untrained=True: true llm-full-mp-gpus: inherits: _llm @@ -642,7 +642,7 @@ llm-full-mp-gpus: safetensors=true: true batch_size=2: true gradient_accumulation_steps=1: true - no_pretrained=True: true + untrained=True: true llm-full-mp-nodes: tags: @@ -666,7 +666,7 @@ llm-full-mp-nodes: safetensors=true: true batch_size=2: true gradient_accumulation_steps=1: true - no_pretrained=True: true + untrained=True: true num_machines: 2 requires_capabilities: From f75e3a50e670bea8e44360ea69d9a7a3d4f86a64 Mon Sep 17 00:00:00 2001 From: Satya Ortiz-Gagne Date: Tue, 17 Sep 2024 21:42:25 -0400 Subject: [PATCH 6/8] Add slurm system setup --- config/base.yaml | 7 +- config/cloud-multinodes-system.yaml | 13 + config/cloud-system.yaml | 24 + config/examples/cloud-multinodes-system.yaml | 10 + config/examples/cloud-system.yaml | 11 + docs/usage.rst | 89 +++- milabench/cli/cloud.py | 51 +- milabench/commands/__init__.py | 11 +- milabench/scripts/covalent/__main__.py | 463 ++++++++++++++---- milabench/scripts/covalent/covalent_bashrc.sh | 46 ++ .../scripts/covalent/milabench_bashrc.sh | 31 ++ .../scripts/covalent/python3/__main__.py | 20 + milabench/scripts/covalent/requirements.txt | 3 +- milabench/scripts/utils.py | 15 +- 14 files changed, 669 insertions(+), 125 deletions(-) create mode 100644 milabench/scripts/covalent/covalent_bashrc.sh create mode 100644 milabench/scripts/covalent/milabench_bashrc.sh create mode 100644 milabench/scripts/covalent/python3/__main__.py diff --git a/config/base.yaml b/config/base.yaml index f15333458..2509a5132 100644 --- a/config/base.yaml +++ b/config/base.yaml @@ -28,6 +28,7 @@ _torchvision: --loader: pytorch --data: "{milabench_data}/FakeImageNet" + _torchvision_ddp: inherits: _defaults definition: ../benchmarks/torchvision_ddp @@ -113,6 +114,7 @@ _timm: --dataset: "FakeImageNet" --workers: "auto({n_worker}, 8)" + _accelerate_opt: inherits: _defaults tags: @@ -149,6 +151,7 @@ _accelerate_opt: use_deepspeed: true num_machines: 1 + fp16: inherits: _flops @@ -388,6 +391,7 @@ brax: --num-minibatches: 32 --num-envs: 8192 + _diffusion: inherits: _defaults definition: ../benchmarks/diffusion @@ -530,11 +534,11 @@ _llm: definition: ../benchmarks/llm install_group: torch + llm-lora-single: inherits: _llm plan: method: per_gpu - argv: "{milabench_code}/recipes/lora_finetune_single_device.py": true --config: "{milabench_code}/configs/llama3_8B_lora_single_device.yaml" @@ -596,6 +600,7 @@ llm-lora-ddp-nodes: requires_capabilities: - "len(nodes) >= ${num_machines}" + llm-lora-mp-gpus: inherits: _llm plan: diff --git a/config/cloud-multinodes-system.yaml b/config/cloud-multinodes-system.yaml index 4f7fae391..dd6e8712c 100644 --- a/config/cloud-multinodes-system.yaml +++ b/config/cloud-multinodes-system.yaml @@ -38,3 +38,16 @@ system: size: Standard_NV72ads_A10_v5 location: eastus2 disk_size: 512 + slurm__a100_x2: + address: localhost + bashrc_path: "{bashrc_path}" + remote_workdir: "scratch/cov-{job_uuid}-workdir" + use_srun: null + options: + ntasks-per-node: 1 + gpus-per-task: a100l:2 + cpus-per-task: 12 + time: "3:0:0" + mem: 500000 + partition: short-unkillable + nodelist: cn-g[001-029] diff --git a/config/cloud-system.yaml b/config/cloud-system.yaml index 056ef3640..8401be63c 100644 --- a/config/cloud-system.yaml +++ b/config/cloud-system.yaml @@ -38,3 +38,27 @@ system: size: Standard_NV72ads_A10_v5 location: eastus2 disk_size: 512 + slurm__a100_x1: + address: localhost + bashrc_path: "{bashrc_path}" + remote_workdir: "scratch/cov-{job_uuid}-workdir" + use_srun: null + options: + ntasks-per-node: 1 + gpus-per-task: a100l:1 + cpus-per-task: 6 + time: "3:0:0" + mem: 32000 + partition: unkillable + slurm__a100_x4: + address: localhost + bashrc_path: "{bashrc_path}" + remote_workdir: "scratch/cov-{job_uuid}-workdir" + use_srun: null + options: + ntasks-per-node: 1 + gpus-per-task: a100l:4 + cpus-per-task: 24 + time: "3:0:0" + mem: 1000000 + partition: short-unkillable diff --git a/config/examples/cloud-multinodes-system.yaml b/config/examples/cloud-multinodes-system.yaml index 5066af5eb..27f1e41b1 100644 --- a/config/examples/cloud-multinodes-system.yaml +++ b/config/examples/cloud-multinodes-system.yaml @@ -35,3 +35,13 @@ system: volume_size: 8 region: us-east-2 state_id: 71669879043a3864225aabb94f91a2d4 + slurm: + address: localhost + bashrc_path: "{bashrc_path}" + remote_workdir: "scratch/cov-{job_uuid}-workdir" + use_srun: null + options: + ntasks-per-node: 1 + cpus-per-task: 1 + time: "0:30:0" + mem: 1000 diff --git a/config/examples/cloud-system.yaml b/config/examples/cloud-system.yaml index b3d1f70aa..d3d3942cc 100644 --- a/config/examples/cloud-system.yaml +++ b/config/examples/cloud-system.yaml @@ -28,3 +28,14 @@ system: instance_type: t2.micro volume_size: 8 region: us-east-2 + slurm: + # covalent-slurm-plugin args + address: localhost + bashrc_path: "{bashrc_path}" + remote_workdir: "scratch/cov-{job_uuid}-workdir" + use_srun: null + options: + ntasks-per-node: 1 + cpus-per-task: 1 + time: "0:30:0" + mem: 1000 diff --git a/docs/usage.rst b/docs/usage.rst index b2a25d85d..4f0fa6f13 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -102,7 +102,7 @@ Create a cloud system configuration ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Add a ``cloud_profiles`` section to the ``system`` configuration which lists the -supported cloud profiles. +supported cloud and slurm profiles. .. notes:: @@ -150,14 +150,95 @@ Run milabench on the cloud ^^^^^^^^^^^^^^^^^^^^^^^^^^ 1. | Initialize the cloud instances - | ``milabench cloud --system {{SYSTEM_CONFIG.YAML}} --setup --run-on {{PROFILE}} >{{SYSTEM_CLOUD_CONFIG.YAML}}`` + | ``milabench cloud --setup --system {{SYSTEM_CONFIG.YAML}} --run-on {{PROFILE}} >{{SYSTEM_CLOUD_CONFIG.YAML}}`` 2. | Prepare, install and run milabench | ``milabench [prepare|install|run] --system {{SYSTEM_CLOUD_CONFIG.YAML}}`` 3. | Destroy the cloud instances - | ``milabench teardown --system {{SYSTEM_CLOUD_CONFIG.YAML}} --run-on {{PROFILE}}`` + | ``milabench cloud --teardown --system {{SYSTEM_CLOUD_CONFIG.YAML}} --run-on {{PROFILE}}`` | or - | ``milabench teardown --system {{SYSTEM_CLOUD_CONFIG.YAML}} --run-on {{PLATFORM}} --all`` + | ``milabench cloud --teardown --system {{SYSTEM_CLOUD_CONFIG.YAML}} --run-on {{PLATFORM}} --all`` | to destroy not just a single cloud instance but all instances on a specified platform that were instanced from the current local machine + + +Use milabench on slurm +~~~~~~~~~~~~~~~~~~~~~~ + + +Create a slurm system configuration +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Add a ``cloud_profiles`` section to the ``system`` configuration which lists the +supported cloud and slurm profiles. + +.. notes:: + + Nodes that should be created on the cloud should have the ``1.1.1.1`` ip + address placeholder. Other ip addresses will be used as-is and no cloud + instance will be created for that node + +.. notes:: + + A cloud profile entry needs to start with a covalent plugin (e.g. `slurm`). To + define multiple profiles on the same cloud platform, use the form + ``{PLATFORM}__{PROFILE_NAME}`` (e.g. ``slurm__profile``). All cloud profile + attributes will be used as is as argument for the target covalent plugin + +.. code-block:: yaml + + system: + nodes: + - name: manager + # Use 1.1.1.1 as an ip placeholder + ip: 1.1.1.1 + main: true + user: + - name: node1 + ip: 1.1.1.1 + main: false + user: + + # Cloud instances profiles + cloud_profiles: + # The cloud platform to use in the form of {PLATFORM} or + # {PLATFORM}__{PROFILE_NAME} + slurm: + username: usename + address: localhost + ssh_key_file: ssh_key_file + # bashrc_path will be replaced by the content of + # milabench/scripts/covalent/covalent_bashrc.sh + bashrc_path: "{bashrc_path}" + # job_uuid will be replaced by the generated job's uuid + remote_workdir: "cov-{job_uuid}-workdir" + use_srun: null + options: + ntasks-per-node: 1 + cpus-per-task: 1 + time: "0:30:0" + mem: 1000 + + +Run milabench on slurm +^^^^^^^^^^^^^^^^^^^^^^ + +1. | Initialize the slurm instances + | ``milabench cloud --setup --system {{SYSTEM_CONFIG.YAML}} --run-on {{PROFILE}} >{{SYSTEM_SLURM_CONFIG.YAML}}`` + +2. | Prepare, install and run milabench + | ``milabench [prepare|install|run] --system {{SYSTEM_SLURM_CONFIG.YAML}}`` + +3. | Destroy the slurm instances + | ``milabench cloud --teardown --system {{SYSTEM_SLURM_CONFIG.YAML}} --run-on {{PROFILE}}`` + +.. notes:: + + Because the milabench's path is expected to be the same on local machine and + the remote machine, it's currently necessary to run the commands from the + slurm cluster. As the ``milabench cloud --[setup|teardown]`` commands requires + a covalent server to run and to avoid overloading the login nodes resources, + it's preferable to request a cpu compute node which will host to the covalent + server. An allocation with minimal resources like ``--nodes 1 --cpus-per-task + 1 --mem 2000`` should be enough. diff --git a/milabench/cli/cloud.py b/milabench/cli/cloud.py index 859cdab87..1d89b69e4 100644 --- a/milabench/cli/cloud.py +++ b/milabench/cli/cloud.py @@ -60,29 +60,33 @@ def manage_cloud(pack, run_on, action="setup"): "private_ip":(lambda v: ("internal_ip",v)), "username":(lambda v: ("user",v)), "ssh_key_file":(lambda v: ("key",v)), - # "env":(lambda v: ("env",[".", v, ";", "conda", "activate", "milabench", "&&"])), + "env":(lambda v: ("env",[".", v, "milabench", "&&"])), + "slurm_job_id":(lambda v: ("slurm_job_id",v)), } - plan_params = deepcopy(pack.config["system"]["cloud_profiles"][run_on]) + plan_params = pack.config["system"]["cloud_profiles"][run_on] run_on, *profile = run_on.split("__") profile = profile[0] if profile else "" default_state_prefix = profile or run_on default_state_id = "_".join((pack.config["hash"][:6], blabla())) - local_base = pack.dirs.base.absolute() - local_data_dir = _get_common_dir(ROOT_FOLDER.parent, local_base.parent) - if local_data_dir is None: - local_data_dir = local_base.parent - remote_data_dir = XPath("/data") / local_data_dir.name + plan_params["state_prefix"] = plan_params.get("state_prefix", default_state_prefix) + plan_params["state_id"] = plan_params.get("state_id", default_state_id) + plan_params["keep_alive"] = None + + # local_base = pack.dirs.base.absolute() + # local_data_dir = _get_common_dir(ROOT_FOLDER.parent, local_base.parent) + # if local_data_dir is None: + # local_data_dir = local_base.parent + # remote_data_dir = XPath("/data") / local_data_dir.name + + plan_params_copy = deepcopy(plan_params) nodes = iter(enumerate(pack.config["system"]["nodes"])) for i, n in nodes: - if n["ip"] != "1.1.1.1": + if n["ip"] != "1.1.1.1" and action == _SETUP: continue - plan_params["state_prefix"] = plan_params.get("state_prefix", default_state_prefix) - plan_params["state_id"] = plan_params.get("state_id", default_state_id) - plan_params["cluster_size"] = max(len(pack.config["system"]["nodes"]), i + 1) - plan_params["keep_alive"] = None + plan_params_copy["cluster_size"] = max(len(pack.config["system"]["nodes"]), i + 1) import milabench.scripts.covalent as cv @@ -101,17 +105,17 @@ def manage_cloud(pack, run_on, action="setup"): "-m", cv.__name__, run_on, f"--{action}", - *_flatten_cli_args(**plan_params) + *_flatten_cli_args(**plan_params_copy) ] - if action == _SETUP: - cmd += [ - "--", - "bash", "-c", - _or_sudo(f"mkdir -p '{local_data_dir.parent}'") + - " && " + _or_sudo(f"chmod a+rwX '{local_data_dir.parent}'") + - f" && mkdir -p '{remote_data_dir}'" - f" && ln -sfT '{remote_data_dir}' '{local_data_dir}'" - ] + # if action == _SETUP: + # cmd += [ + # "--", + # "bash", "-c", + # _or_sudo(f"mkdir -p '{local_data_dir.parent}'") + + # " && " + _or_sudo(f"chmod a+rwX '{local_data_dir.parent}'") + + # f" && mkdir -p '{remote_data_dir}'" + # f" && ln -sfT '{remote_data_dir}' '{local_data_dir}'" + # ] p = subprocess.Popen( cmd, stdout=subprocess.PIPE, @@ -155,6 +159,9 @@ def manage_cloud(pack, run_on, action="setup"): stderr ) + if action == _TEARDOWN: + break + return pack.config["system"] diff --git a/milabench/commands/__init__.py b/milabench/commands/__init__.py index ad281b23b..fcd6ccbf4 100644 --- a/milabench/commands/__init__.py +++ b/milabench/commands/__init__.py @@ -450,6 +450,11 @@ def _find_node_config(self) -> Dict: return n return {} + def _load_env(self, node): + if node.get("env", None): + return node["env"] + return [] + def is_local(self): localnode = self.pack.config["system"]["self"] @@ -484,7 +489,7 @@ def _argv(self, **kwargs) -> List: argv.append(f"-p{self.port}") argv.append(host) - return argv # + ["env", "-i"] + return argv + self._load_env(node) class SCPCommand(SSHCommand, CmdCommand): @@ -505,6 +510,10 @@ def __init__( self.src = src self.dest = dest if dest is not None else self.src + def _load_env(self, node): + del node + return [] + def _argv(self, **kwargs) -> List: argv = super()._argv(**kwargs) diff --git a/milabench/scripts/covalent/__main__.py b/milabench/scripts/covalent/__main__.py index d4de9d932..689eda8fa 100644 --- a/milabench/scripts/covalent/__main__.py +++ b/milabench/scripts/covalent/__main__.py @@ -1,9 +1,50 @@ import argparse +import ast import os import pathlib import subprocess import sys import tempfile +from time import sleep +import uuid + + +def _arg_pop(args:argparse.Namespace, key:str): + value = args.__getattribute__(key) + args.__delattr__(key) + return value + +ARGS_DEFAULT_SETUP = { + "slurm": { + "state_prefix": {}, + "state_id": {}, + "cluster_size": {}, + "keep_alive": {"action": "store_true"}, + } +} + +ARGS_MAP = { + "slurm": { + "state_prefix": lambda args, k:_arg_pop(args, k), + "state_id": lambda args, k:args.options.setdefault("job-name", _arg_pop(args, k)), + "cluster_size": lambda args, k:args.options.setdefault("nodes", _arg_pop(args, k)), + "keep_alive": lambda args, k:_arg_pop(args, k), + } +} + +_SETUP = {} + +_TEARDOWN = {} + +_CONNECTION_ATTRIBUTES = { + "hostname": None, + "username": None, + "ssh_key_file": None, + "private_ip": None, + "env": None, + "python_path": None, + "slurm_job_id": None +} def serve(*argv): @@ -19,69 +60,328 @@ def _get_executor_kwargs(args): } -def executor(executor_cls, args, *argv): +def _wait_for_any(*dispatch_ids): import covalent as ct - def _popen(cmd, *args, _env=None, **kwargs): - _env = _env if _env is not None else {} + dispatch_ids = set(dispatch_ids) + while True: + for dispatch_id in set(dispatch_ids): + status = ct.get_result( + dispatch_id=dispatch_id, + wait=False, + status_only=True + )["status"] + if status in [ct.status.COMPLETED]: + yield dispatch_id + dispatch_ids.remove(dispatch_id) + elif status in [ct.status.FAILED, ct.status.CANCELLED]: + raise RuntimeError(f"Job {dispatch_id} failed") + sleep(5) + + +def _format(lines:list, **template_kv): + for l in lines: + for k, v in template_kv.items(): + if "{{" + k + "}}" in l: + yield l[:l.find("{{")] + v + l[l.find("}}")+2:] + break + else: + yield l - for envvar in _env.keys(): - envvar_val = _env[envvar] - if not envvar_val: - continue +def _popen(cmd, *args, _env=None, **kwargs): + _env = _env if _env is not None else {} - envvar_val = pathlib.Path(envvar_val).expanduser() - if str(envvar_val) != _env[envvar]: - _env[envvar] = str(envvar_val) + for envvar in _env.keys(): + envvar_val = _env[envvar] - if "MILABENCH_CONFIG_CONTENT" in _env: - _config_dir = pathlib.Path(_env["MILABENCH_CONFIG"]).parent - with tempfile.NamedTemporaryFile("wt", dir=str(_config_dir), suffix=".yaml", delete=False) as _f: - _f.write(_env["MILABENCH_CONFIG_CONTENT"]) - _env["MILABENCH_CONFIG"] = _f.name + if not envvar_val: + continue - try: - cmd = (str(pathlib.Path(cmd[0]).expanduser()), *cmd[1:]) - except IndexError: - pass - - cwd = kwargs.pop("cwd", None) - if cwd is not None: - cwd = str(pathlib.Path(cwd).expanduser()) - kwargs["cwd"] = cwd - - _env = {**os.environ.copy(), **kwargs.pop("env", {}), **_env} - - kwargs = { - **kwargs, - "env": _env, - "stdout": subprocess.PIPE, - "stderr": subprocess.PIPE, - } - p = subprocess.Popen(cmd, *args, **kwargs) + envvar_val = pathlib.Path(envvar_val).expanduser() + if str(envvar_val) != _env[envvar]: + _env[envvar] = str(envvar_val) + + if "MILABENCH_CONFIG_CONTENT" in _env: + _config_dir = pathlib.Path(_env["MILABENCH_CONFIG"]).parent + with tempfile.NamedTemporaryFile("wt", dir=str(_config_dir), suffix=".yaml", delete=False) as _f: + _f.write(_env["MILABENCH_CONFIG_CONTENT"]) + _env["MILABENCH_CONFIG"] = _f.name + + try: + cmd = (str(pathlib.Path(cmd[0]).expanduser()), *cmd[1:]) + except IndexError: + pass + + cwd = kwargs.pop("cwd", None) + if cwd is not None: + cwd = str(pathlib.Path(cwd).expanduser()) + kwargs["cwd"] = cwd + + _env = {**os.environ.copy(), **kwargs.pop("env", {}), **_env} + + kwargs = { + **kwargs, + "env": _env, + "stdout": subprocess.PIPE, + "stderr": subprocess.PIPE, + } + p = subprocess.Popen(cmd, *args, **kwargs) + + stdout_chunks = [] + while True: + line = p.stdout.readline() + if not line: + break + line_str = line.decode("utf-8").strip() + stdout_chunks.append(line_str) + print(line_str) + + _, stderr = p.communicate() + stderr = stderr.decode("utf-8").strip() + stdout = os.linesep.join(stdout_chunks) + + if p.returncode != 0: + raise subprocess.CalledProcessError( + p.returncode, + (cmd, args, kwargs), + stdout, + stderr + ) + return p.returncode, stdout, stderr + + +def _setup_terraform(executor:"ct.executor.BaseExecutor"): + import covalent as ct + + result = ct.dispatch_sync( + ct.lattice(executor.get_connection_attributes) + )().result + + assert result and result[0] + + all_connection_attributes, _ = result + master_host:str = next(iter(all_connection_attributes)) + + if len(all_connection_attributes) > 1: + # Add master node to known host to avoid unknown host error The + # authenticity of host '[hostname] ([IP address])' can't be established. + new_host = subprocess.run( + ["ssh-keyscan", master_host], + stdout=subprocess.PIPE, + check=True + ).stdout.decode("utf8") + known_hosts = pathlib.Path("~/.ssh/known_hosts").expanduser() + with known_hosts.open("at") as _f: + _f.write(new_host) + + # Add ssh file to master node to allow connections to worker nodes + ssh_key_file = all_connection_attributes[master_host]["ssh_key_file"] + fn = pathlib.Path(ssh_key_file) + result = ct.dispatch_sync( + ct.lattice(executor.cp_to_remote) + )(f".ssh/{fn.name.split('.')[0]}", str(fn)) + + assert result.status == ct.status.COMPLETED + + return all_connection_attributes + + +def _teardown_terraform(executor:"ct.executor.BaseExecutor"): + result = executor.stop_cloud_instance().result + assert result is not None + + +def _slurm_executor(executor:"ct.executor.SlurmExecutor", job_uuid:uuid.UUID): + import covalent as ct + + _executor = ct.executor.SlurmExecutor() + _executor.from_dict(executor.to_dict()) + + executor = _executor + executor.conda_env = executor.conda_env or "covalent" + bashrc_path = f"""'' +{pathlib.Path(__file__).with_name("covalent_bashrc.sh").read_text()} +""" + bashrc_path = "\n".join( + _format( + bashrc_path.splitlines(), + conda_env=executor.conda_env, + python_version=f"{sys.version_info.major}.{sys.version_info.minor}", + covalent_version=ct.__version__, + ) + ) + executor.bashrc_path = executor.bashrc_path or "{bashrc_path}" + if "{bashrc_path}" in executor.bashrc_path: + executor.bashrc_path = executor.bashrc_path.format(bashrc_path=bashrc_path) + executor.remote_workdir = executor.remote_workdir or "cov-{job_uuid}-workdir" + executor.remote_workdir = executor.remote_workdir.format(job_uuid=job_uuid) + executor.options["job-name"] = ( + executor.options.get("job-name", None) or f"cov-{job_uuid}" + ) + + return executor - stdout_chunks = [] + +def _setup_slurm(executor:"ct.executor.SlurmExecutor"): + import covalent as ct + + job_uuid = uuid.uuid4() + job_file = f"covalent_job_{job_uuid}" + + _executor = ct.executor.SlurmExecutor() + _executor.from_dict(executor.to_dict()) + + executor = _slurm_executor(executor, job_uuid) + + job_connection_executor = ct.executor.SlurmExecutor() + job_connection_executor.from_dict(executor.to_dict()) + # Store job connection attributes + job_connection_executor.prerun_commands = f""" +# print connection attributes +printenv | + grep -E ".*SLURM.*NODENAME|.*SLURM.*JOB_ID" | + sort -u >>"{job_file}" && +srun printenv | + grep -E ".*SLURM.*NODENAME|.*SLURM.*JOB_ID" | + sort -u >>"{job_file}" && +echo "USERNAME=$USER" >>"{job_file}" && +echo "{job_uuid}" >>"{job_file}" +""".splitlines() + + query_executor = ct.executor.SlurmExecutor() + query_executor.from_dict(executor.to_dict()) + query_executor.options = { + "nodes": 1, + "cpus-per-task": 1, + "mem": 1000, + "job-name": executor.options["job-name"], + } + + @ct.electron() + def _empty(): + pass + + @ct.electron() + def _keep_alive(): while True: - line = p.stdout.readline() - if not line: - break - line_str = line.decode("utf-8").strip() - stdout_chunks.append(line_str) - print(line_str) - - _, stderr = p.communicate() - stderr = stderr.decode("utf-8").strip() - stdout = os.linesep.join(stdout_chunks) - - if p.returncode != 0: - raise subprocess.CalledProcessError( - p.returncode, - (cmd, args, kwargs), - stdout, - stderr + sleep(60) + + @ct.electron() + def _query_connection_attributes(milabench_bashrc:str=""): + _job_file = pathlib.Path(job_file).expanduser() + _job_file.touch() + content = _job_file.read_text().splitlines() + while (not content or content[-1].strip() != f"{job_uuid}"): + sleep(5) + content = _job_file.read_text().splitlines() + + nodes = [] + connection_attributes = _CONNECTION_ATTRIBUTES.copy() + + milabench_bashrc = "\n".join( + _format( + milabench_bashrc.splitlines(), + milabench_env="cov-slurm-milabench", + python_version=f"{sys.version_info.major}.{sys.version_info.minor}", ) - return p.returncode, stdout, stderr + ) + if milabench_bashrc: + milabench_bashrc_file = _job_file.with_name("milabench_bashrc.sh").resolve() + milabench_bashrc_file.write_text(milabench_bashrc) + connection_attributes["env"] = str(milabench_bashrc_file) + + for l in _job_file.read_text().splitlines(): + try: + key, value = l.strip().split("=") + except ValueError: + # end flag + break + if "NODENAME" in key and value not in nodes: + nodes.append(value) + elif "USERNAME" in key: + connection_attributes["username"] = value + elif "JOB_ID" in key: + connection_attributes["slurm_job_id"] = value + + return { + hostname: { + **connection_attributes, + **{ + "hostname": hostname, + "private_ip": hostname, + }, + } + for hostname in nodes + } + + try: + # setup covalent for jobs + next(_wait_for_any(ct.dispatch(ct.lattice(_empty, executor=query_executor))())) + # setup nodes and retrieve connection attributes + job_dispatch_id = ct.dispatch( + ct.lattice( + lambda:_keep_alive(), + executor=job_connection_executor + ), + disable_run=False + )() + query_dispatch_id = ct.dispatch( + ct.lattice( + _query_connection_attributes, + executor=query_executor + ), + disable_run=False + )( + milabench_bashrc=pathlib.Path(__file__).with_name("milabench_bashrc.sh").read_text() + ) + next(_wait_for_any(job_dispatch_id, query_dispatch_id)) + all_connection_attributes = ct.get_result( + dispatch_id=query_dispatch_id, + wait=False + ).result + + assert all_connection_attributes + + except: + _teardown_slurm(query_executor) + raise + + return all_connection_attributes + + +def _teardown_slurm(executor:"ct.executor.SlurmExecutor"): + import covalent as ct + + @ct.electron() + def _empty(): + pass + + assert executor.options["job-name"], "Jobs to teardown must have an explicit name" + + _exec = _slurm_executor(executor, "DELETE") + _exec.options = { + "nodes": 1, + "cpus-per-task": 1, + "mem": 1000, + "job-name": executor.options["job-name"], + } + _exec.prerun_commands = f""" +# cancel jobs +scancel --jobname="{_exec.options['job-name']}" +""".splitlines() + ct.dispatch_sync(ct.lattice(_empty, executor=_exec))() + + +def executor(executor_cls, args:argparse.Namespace, *argv): + import covalent as ct + + _SETUP[ct.executor.AzureExecutor] = _setup_terraform + _SETUP[ct.executor.EC2Executor] = _setup_terraform + _SETUP[ct.executor.SlurmExecutor] = _setup_slurm + _TEARDOWN[ct.executor.AzureExecutor] = _teardown_terraform + _TEARDOWN[ct.executor.EC2Executor] = _teardown_terraform + _TEARDOWN[ct.executor.SlurmExecutor] = _teardown_slurm executor:ct.executor.BaseExecutor = executor_cls( **_get_executor_kwargs(args), @@ -89,45 +389,13 @@ def _popen(cmd, *args, _env=None, **kwargs): return_code = 0 try: if args.setup: - result = ct.dispatch_sync( - ct.lattice(executor.get_connection_attributes) - )().result - - assert result and result[0] - - all_connection_attributes, _ = result - master_host:str = None - for hostname, connection_attributes in all_connection_attributes.items(): + for hostname, connection_attributes in _SETUP[executor_cls](executor).items(): print(f"hostname::>{hostname}") for attribute,value in connection_attributes.items(): - if attribute == "hostname": + if attribute == "hostname" or value is None: continue print(f"{attribute}::>{value}") - master_host = master_host or hostname - - if len(all_connection_attributes) > 1: - # Add master node to known host to avoid unknown host error - # The authenticity of host '[hostname] ([IP address])' can't be established. - new_host = subprocess.run( - ["ssh-keyscan", master_host], - stdout=subprocess.PIPE, - check=True - ).stdout.decode("utf8") - known_hosts = pathlib.Path("~/.ssh/known_hosts").expanduser() - with known_hosts.open("at") as _f: - _f.write(new_host) - - # Add ssh file to master node to allow connections to worker - # nodes - ssh_key_file = all_connection_attributes[master_host]["ssh_key_file"] - fn = pathlib.Path(ssh_key_file) - result = ct.dispatch_sync( - ct.lattice(executor.cp_to_remote) - )(f".ssh/{fn.name.split('.')[0]}", str(fn)) - - assert result.status == ct.status.COMPLETED - if argv: result = ct.dispatch_sync( ct.lattice(executor.list_running_instances) @@ -166,8 +434,7 @@ def _popen(cmd, *args, _env=None, **kwargs): finally: if args.teardown: - result = executor.stop_cloud_instance().result - assert result is not None + _TEARDOWN[executor_cls](executor) return return_code @@ -187,7 +454,7 @@ def main(argv=None): subparsers = parser.add_subparsers() subparser = subparsers.add_parser("serve") subparser.add_argument(f"argv", nargs=argparse.REMAINDER) - for p in ("azure","ec2"): + for p in ("azure", "ec2", "slurm"): try: config = ct.get_config(f"executors.{p}") except KeyError: @@ -201,10 +468,21 @@ def main(argv=None): add_argument_kwargs = {} if isinstance(default, bool): add_argument_kwargs["action"] = "store_false" if default else "store_true" + elif any(isinstance(default, t) for t in [dict, list]): + add_argument_kwargs["type"] = ast.literal_eval + add_argument_kwargs["default"] = str(default) else: add_argument_kwargs["default"] = default subparser.add_argument(f"--{param.replace('_', '-')}", **add_argument_kwargs) + for param, add_argument_kwargs in ARGS_DEFAULT_SETUP.get(p, {}).items(): + if param in config: + raise ValueError( + f"Found existing argument {param} in both {config} and" + f" {ARGS_DEFAULT_SETUP}" + ) + subparser.add_argument(f"--{param.replace('_', '-')}", **add_argument_kwargs) + try: cv_argv, argv = argv[:argv.index("--")], argv[argv.index("--")+1:] except ValueError: @@ -212,6 +490,9 @@ def main(argv=None): args = parser.parse_args(cv_argv) + for arg, _map in ARGS_MAP.get(cv_argv[0], {}).items(): + _map(args, arg) + if cv_argv[0] == "serve": assert not argv return serve(*args.argv) @@ -219,6 +500,8 @@ def main(argv=None): executor_cls = ct.executor.AzureExecutor elif cv_argv[0] == "ec2": executor_cls = ct.executor.EC2Executor + elif cv_argv[0] == "slurm": + executor_cls = ct.executor.SlurmExecutor else: raise diff --git a/milabench/scripts/covalent/covalent_bashrc.sh b/milabench/scripts/covalent/covalent_bashrc.sh new file mode 100644 index 000000000..197620c1c --- /dev/null +++ b/milabench/scripts/covalent/covalent_bashrc.sh @@ -0,0 +1,46 @@ +function _get_options { + set +o | cut -d' ' -f2- | while read set_option + do + echo "${set_option}" + done +} + +function _set_options { + while [[ $# -gt 0 ]] + do + local _arg="$1"; shift + case "${_arg}" in + -o) set -o "$1"; shift ;; + +o) set +o "$1"; shift ;; + -h | --help | *) + exit 1 + ;; + esac + done +} + +_options=$(_get_options) +set -o errexit -o pipefail + +_NAME="{{conda_env}}" +conda --version >&2 2>/dev/null || module load anaconda/3 +# {{python_version}} needs to be formatted +conda activate ${_NAME} || conda create -y -n ${_NAME} "python={{python_version}}" virtualenv +conda activate ${_NAME} + +function conda { + echo "DEACTIVATED conda" "$@" >&2 +} + +if [ ! -f venv/bin/activate ] +then + python3 -m virtualenv venv/ + . venv/bin/activate + # {{covalent_version}} needs to be formatted + python3 -m pip install "covalent=={{covalent_version}}" +else + . venv/bin/activate +fi + +_set_options $_options +unset _options diff --git a/milabench/scripts/covalent/milabench_bashrc.sh b/milabench/scripts/covalent/milabench_bashrc.sh new file mode 100644 index 000000000..1ee96d788 --- /dev/null +++ b/milabench/scripts/covalent/milabench_bashrc.sh @@ -0,0 +1,31 @@ +function _get_options { + set +o | cut -d' ' -f2- | while read set_option + do + echo "${set_option}" + done +} + +function _set_options { + while [[ $# -gt 0 ]] + do + local _arg="$1"; shift + case "${_arg}" in + -o) set -o "$1"; shift ;; + +o) set +o "$1"; shift ;; + -h | --help | *) + exit 1 + ;; + esac + done +} + +_options=$(_get_options) +set -o errexit -o pipefail + +_NAME="{{milabench_env}}" +conda --version >&2 2>/dev/null || module load anaconda/3 +conda activate ${_NAME} || conda create -y -n ${_NAME} "python={{python_version}}" virtualenv +conda activate ${_NAME} + +_set_options $_options +unset _options diff --git a/milabench/scripts/covalent/python3/__main__.py b/milabench/scripts/covalent/python3/__main__.py new file mode 100644 index 000000000..186d9a170 --- /dev/null +++ b/milabench/scripts/covalent/python3/__main__.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 + +import subprocess +import sys + + +def main(argv=None): + if argv is None: + argv = sys.argv[1:] + + from ...utils import get_module_venv + from .. import __main__ + check_if_module = "import covalent" + python3, env = get_module_venv(__main__.__file__, check_if_module) + + return subprocess.call([python3, *argv], env=env) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/milabench/scripts/covalent/requirements.txt b/milabench/scripts/covalent/requirements.txt index b70efc793..4b322dc3f 100644 --- a/milabench/scripts/covalent/requirements.txt +++ b/milabench/scripts/covalent/requirements.txt @@ -1,3 +1,4 @@ covalent==0.232 covalent-ec2-plugin @ git+https://github.com/satyaog/covalent-ec2-plugin.git@feature/milabench -covalent-azure-plugin @ git+https://github.com/satyaog/covalent-azure-plugin.git@feature/milabench \ No newline at end of file +covalent-azure-plugin @ git+https://github.com/satyaog/covalent-azure-plugin.git@feature/milabench +covalent-slurm-plugin diff --git a/milabench/scripts/utils.py b/milabench/scripts/utils.py index 5aec72d06..022481089 100644 --- a/milabench/scripts/utils.py +++ b/milabench/scripts/utils.py @@ -1,3 +1,4 @@ +import getpass import json import pathlib import subprocess @@ -16,9 +17,9 @@ def get_venv(venv:pathlib.Path) -> dict: return json.loads(env) -def run_in_module_venv(module_main:str, check_if_module:str, argv:list=None): +def get_module_venv(module_main:str, check_if_module:str): module = pathlib.Path(module_main).resolve().parent - cache_dir = pathlib.Path(f"/tmp/milabench/{module.name}_venv") + cache_dir = pathlib.Path(f"/tmp/{getpass.getuser()}/milabench/{module.name}_venv") python3 = str(cache_dir / "bin/python3") try: subprocess.run([python3, "-c", check_if_module], check=True, @@ -38,7 +39,9 @@ def run_in_module_venv(module_main:str, check_if_module:str, argv:list=None): str(module / "requirements.txt") ], stdout=sys.stderr, check=True) subprocess.run([python3, "-c", check_if_module], check=True, stdout=sys.stderr) - return subprocess.call( - [python3, module_main, *argv], - env=get_venv(cache_dir) - ) \ No newline at end of file + return python3, get_venv(cache_dir) + + +def run_in_module_venv(module_main:str, check_if_module:str, argv:list=None): + python3, env = get_module_venv(module_main, check_if_module) + return subprocess.call([python3, module_main, *argv], env=env) \ No newline at end of file From 7ff4db3dd08f47d3c83cfd07a92cba47b3517e68 Mon Sep 17 00:00:00 2001 From: Satya Ortiz-Gagne Date: Fri, 4 Oct 2024 09:38:06 -0400 Subject: [PATCH 7/8] Update ci --- .github/workflows/cloud-ci.yml | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/.github/workflows/cloud-ci.yml b/.github/workflows/cloud-ci.yml index e58760a08..c767cda1d 100644 --- a/.github/workflows/cloud-ci.yml +++ b/.github/workflows/cloud-ci.yml @@ -45,7 +45,7 @@ jobs: ARM_TENANT_ID: "${{ secrets.ARM_TENANT_ID }}" ARM_SUBSCRIPTION_ID: "${{ secrets.ARM_SUBSCRIPTION_ID }}" AZURE_CORE_OUTPUT: none - _MULTI_GPUS: "diffusion-gpus,dinov2-giant-gpus,lightning-gpus,llava-gpus,resnet152-ddp-gpus,llm-full-mp-gpus,llm-lora-ddp-gpus,llm-lora-mp-gpus" + _MULTI_GPUS: "multigpu" _MULTI_NODES: "multinode" steps: @@ -113,7 +113,7 @@ jobs: case "$gpus" in "1g") RUN_ON="azure__a100" - EXCLUDE="$EXCLUDE,$_MULTI_GPUS,$_MULTI_NODES" + EXCLUDE="$EXCLUDE,$_MULTI_GPUS" ;; "2g") RUN_ON="azure__a100_x2" @@ -159,6 +159,12 @@ jobs: echo "SELECT=$SELECT" >>$GITHUB_ENV echo "EXCLUDE=$EXCLUDE" >>$GITHUB_ENV + - name: DEBUG covalent logs + if: always() + run: | + cat ~/.cache/covalent/covalent_ui.log + echo >~/.cache/covalent/covalent_ui.log + - name: install benchmarks run: | poetry run milabench install --variant ${{ matrix.arch }} $SELECT $EXCLUDE @@ -184,21 +190,18 @@ jobs: - name: DEBUG state file if: always() run: | - cat /tmp/milabench/covalent_venv/lib/python*/site-packages/covalent_azure_plugin/infra/*.tfstate + cat /tmp/runner/milabench/covalent_venv/lib/python*/site-packages/covalent_azure_plugin/infra/*.tfstate - name: teardown cloud if: always() run: | - if [[ -f "${MILABENCH_SYSTEM%.*}" ]] - then - export MILABENCH_SYSTEM=${MILABENCH_SYSTEM%.*} - fi poetry run milabench cloud \ --teardown \ --run-on $RUN_ON \ --all - - name: DEBUG logs + - name: DEBUG covalent logs if: always() run: | cat ~/.cache/covalent/covalent_ui.log + echo >~/.cache/covalent/covalent_ui.log From 2c1375266192ffad2ac391fa02084ac8a7b83089 Mon Sep 17 00:00:00 2001 From: Satya Ortiz-Gagne Date: Mon, 7 Oct 2024 09:42:05 -0400 Subject: [PATCH 8/8] Partial revert of monitor_tags --- milabench/config.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/milabench/config.py b/milabench/config.py index 6f42dbf8d..4936054dc 100644 --- a/milabench/config.py +++ b/milabench/config.py @@ -12,8 +12,6 @@ config_global = contextvars.ContextVar("config", default=None) execution_count = (0, 0) -_MONITOR_TAGS = {"monogpu", "multigpu", "multinode"} - def set_run_count(total_run, total_bench): global execution_count @@ -93,13 +91,6 @@ def finalize_config(name, bench_config): pack = (XPath(bench_config["config_base"]) / pack).resolve() bench_config["definition"] = str(pack) - if not name.startswith("_") and name != "*": - _tags = set(bench_config["tags"]) - _monitor_tags = _tags & _MONITOR_TAGS - assert len(_monitor_tags) == 1, ( - f"Bench {name} should have exactly one monitor tag. Found {_monitor_tags}" - ) - bench_config["tag"] = [bench_config["name"]] bench_config = OmegaConf.to_object(OmegaConf.create(bench_config))