diff --git a/cm-mlops/script/app-loadgen-generic-python/README.md b/cm-mlops/script/app-loadgen-generic-python/README.md index c453cecc3..443404358 100644 --- a/cm-mlops/script/app-loadgen-generic-python/README.md +++ b/cm-mlops/script/app-loadgen-generic-python/README.md @@ -119,6 +119,15 @@ ___
Click here to expand this section. + * `_cmc` + - Environment variables: + - *CM_CUSTOM_MODEL_CMC*: `True` + - Workflow: + * `_custom,cmc` + - Workflow: + 1. ***Read "deps" on other CM scripts*** + * get,ml-model,cmc + - *Warning: no scripts found* * `_custom,huggingface` - Workflow: 1. ***Read "deps" on other CM scripts*** diff --git a/cm-mlops/script/app-loadgen-generic-python/_cm.yaml b/cm-mlops/script/app-loadgen-generic-python/_cm.yaml index 4019bc044..27468b32d 100644 --- a/cm-mlops/script/app-loadgen-generic-python/_cm.yaml +++ b/cm-mlops/script/app-loadgen-generic-python/_cm.yaml @@ -252,6 +252,17 @@ variations: CM_ML_MODEL_STUB: "#" + cmc: + env: + CM_CUSTOM_MODEL_CMC: yes + + + custom,cmc: + deps: + - tags: get,ml-model,cmc + names: + - cmc-model + input_description: modelpath: diff --git a/cm-mlops/script/app-loadgen-generic-python/src/backend_pytorch.py b/cm-mlops/script/app-loadgen-generic-python/src/backend_pytorch.py index 3182b2488..1fef350b4 100644 --- a/cm-mlops/script/app-loadgen-generic-python/src/backend_pytorch.py +++ b/cm-mlops/script/app-loadgen-generic-python/src/backend_pytorch.py @@ -59,7 +59,6 @@ def create(self) -> Model: print ('') print ('Loading model: {}'.format(self.model_path)) - if self.execution_provider == 'CPUExecutionProvider': torch_provider = 'cpu' elif self.execution_provider == 'CUDAExecutionProvider': @@ -87,17 +86,25 @@ def create(self) -> Model: if not os.path.isfile(cm_model_module): raise Exception('cm.py interface for a PyTorch model was not found in {}'.format(self.model_code)) + print ('') + print ('Collective Mind Connector for the model found: {}'.format(cm_model_module)) + + # Load CM interface for the model import sys sys.path.insert(0, self.model_code) - model_module=importlib.import_module('cm') + model_module=importlib.import_module('cmc') del(sys.path[0]) # Init model if len(self.model_cfg)>0: print ('Model cfg: {}'.format(self.model_cfg)) - model = model_module.model_init(checkpoint, self.model_cfg) + r = model_module.model_init(checkpoint, self.model_cfg) + if r['return']>0: + raise Exception('Error: {}'.format(r['error'])) + + model = r['model'] if torch_provider=='cuda': model.cuda() diff --git a/cm-mlops/script/get-generic-python-lib/README.md b/cm-mlops/script/get-generic-python-lib/README.md index 440b0787a..d521e60cb 100644 --- a/cm-mlops/script/get-generic-python-lib/README.md +++ b/cm-mlops/script/get-generic-python-lib/README.md @@ -40,13 +40,13 @@ Maintainers: [Public MLCommons Task Force on Automation and Reproducibility](htt `cm run script --tags=get,generic-python-lib` -`cm run script --tags=get,generic-python-lib[,variations] ` +`cm run script --tags=get,generic-python-lib[,variations] [--input_flags]` *or* `cmr "get generic-python-lib"` -`cmr "get generic-python-lib [variations]" ` +`cmr "get generic-python-lib [variations]" [--input_flags]` * *See the list of `variations` [here](#variations) and check the [Gettings Started Guide](https://github.com/mlcommons/ck/blob/dev/docs/getting-started.md) for more details.* @@ -85,7 +85,7 @@ Use this [online GUI](https://cKnowledge.org/cm-gui/?tags=get,generic-python-lib #### Run this script via Docker (beta) -`cm docker script "get generic-python-lib[variations]" ` +`cm docker script "get generic-python-lib[variations]" [--input_flags]` ___ ### Customization @@ -487,8 +487,8 @@ ___ - Workflow: * `_torch,rocm` - Environment variables: - - *CM_GENERIC_PYTHON_PIP_INDEX_URL*: `https://download.pytorch.org/whl/rocm5.6` - *CM_GENERIC_PYTHON_PACKAGE_NAME*: `torch` + - *CM_GENERIC_PYTHON_PIP_INDEX_URL*: `https://download.pytorch.org/whl/rocm5.6` - *CM_GENERIC_PYTHON_PIP_UNINSTALL_DEPS*: `torch` - Workflow: 1. ***Read "post_deps" on other CM scripts*** @@ -509,8 +509,8 @@ ___ * `_torch_cuda,pre` - Environment variables: - *CM_GENERIC_PYTHON_PACKAGE_NAME*: `torch` - - *CM_GENERIC_PYTHON_PIP_INDEX_URL*: `https://download.pytorch.org/whl/<<>>` - *CM_GENERIC_PYTHON_PIP_EXTRA*: ` --pre` + - *CM_GENERIC_PYTHON_PIP_INDEX_URL*: `https://download.pytorch.org/whl/<<>>` - *CM_TORCH_VERSION_EXTRA*: `CUDA` - Workflow: 1. ***Read "deps" on other CM scripts*** @@ -532,8 +532,8 @@ ___ - Workflow: * `_torchaudio,rocm` - Environment variables: - - *CM_GENERIC_PYTHON_PIP_INDEX_URL*: `https://download.pytorch.org/whl/rocm5.6` - *CM_GENERIC_PYTHON_PACKAGE_NAME*: `torchaudio` + - *CM_GENERIC_PYTHON_PIP_INDEX_URL*: `https://download.pytorch.org/whl/rocm5.6` - *CM_GENERIC_PYTHON_PIP_UNINSTALL_DEPS*: `torchaudio` - Workflow: * `_torchaudio_cuda` @@ -553,8 +553,8 @@ ___ - Workflow: * `_torchvision,rocm` - Environment variables: - - *CM_GENERIC_PYTHON_PIP_INDEX_URL*: `https://download.pytorch.org/whl/rocm5.6` - *CM_GENERIC_PYTHON_PACKAGE_NAME*: `torchvision` + - *CM_GENERIC_PYTHON_PIP_INDEX_URL*: `https://download.pytorch.org/whl/rocm5.6` - *CM_GENERIC_PYTHON_PIP_UNINSTALL_DEPS*: `torchvision` - Workflow: * `_torchvision_cuda` @@ -614,6 +614,23 @@ ___
+ +#### Script flags mapped to environment +
+Click here to expand this section. + +* `--extra_index_url=value` → `CM_GENERIC_PYTHON_PIP_EXTRA_INDEX_URL=value` +* `--force_install=value` → `CM_TMP_PYTHON_PACKAGE_FORCE_INSTALL=value` +* `--index_url=value` → `CM_GENERIC_PYTHON_PIP_INDEX_URL=value` + +**Above CLI flags can be used in the Python CM API as follows:** + +```python +r=cm.access({... , "extra_index_url":...} +``` + +
+ #### Default environment
@@ -655,7 +672,7 @@ ___ ___ ### Script output -`cmr "get generic-python-lib [,variations]" -j` +`cmr "get generic-python-lib [,variations]" [--input_flags] -j` #### New environment keys (filter) * `CM_PYTHONLIB_*` diff --git a/cm-mlops/script/get-generic-python-lib/_cm.json b/cm-mlops/script/get-generic-python-lib/_cm.json index 5e0992469..b691e85b3 100644 --- a/cm-mlops/script/get-generic-python-lib/_cm.json +++ b/cm-mlops/script/get-generic-python-lib/_cm.json @@ -5,7 +5,6 @@ "cache": true, "category": "Python automation", "clean_files": [], - "tags_help":"get generic-python-lib", "deps": [ { "tags": "detect,os" @@ -18,27 +17,24 @@ "python", "python3" ], - "tags": "get,python3", "skip_if_env": { - "CM_TMP_USE_CUSTOM_PYTHON": [ "on" ] - } + "CM_TMP_USE_CUSTOM_PYTHON": [ + "on" + ] + }, + "tags": "get,python3" }, { "names": [ - "python-pip", "pip" + "python-pip", + "pip" ], - "tags": "get,generic-python-lib,_pip", "skip_if_env": { - "CM_GENERIC_PYTHON_PACKAGE_NAME": [ "pip" ] - } - } - ], - "prehook_deps": [ - { - "tags": "install,onnxruntime,from.src,_cuda", - "enable_if_env": { - "CM_INSTALL_ONNXRUNTIME_GPU_FROM_SRC": [ "yes" ] - } + "CM_GENERIC_PYTHON_PACKAGE_NAME": [ + "pip" + ] + }, + "tags": "get,generic-python-lib,_pip" } ], "extra_cache_tags_from_env": [ @@ -47,11 +43,26 @@ "prefix": "python-" } ], + "input_mapping": { + "extra_index_url": "CM_GENERIC_PYTHON_PIP_EXTRA_INDEX_URL", + "force_install": "CM_TMP_PYTHON_PACKAGE_FORCE_INSTALL", + "index_url": "CM_GENERIC_PYTHON_PIP_INDEX_URL" + }, + "local_env_keys": [ + "CM_GENERIC_PYTHON_PACKAGE_VARIANT" + ], "new_env_keys": [ "CM_PYTHONLIB_*" ], - "local_env_keys": [ - "CM_GENERIC_PYTHON_PACKAGE_VARIANT" + "prehook_deps": [ + { + "enable_if_env": { + "CM_INSTALL_ONNXRUNTIME_GPU_FROM_SRC": [ + "yes" + ] + }, + "tags": "install,onnxruntime,from.src,_cuda" + } ], "tags": [ "get", @@ -59,40 +70,16 @@ "generic", "generic-python-lib" ], + "tags_help": "get generic-python-lib", "uid": "94b62a682bc44791", "variations": { - "package.#": { - "env": { - "CM_GENERIC_PYTHON_PACKAGE_NAME": "#", - "CM_GENERIC_PYTHON_PIP_UNINSTALL_DEPS": "", - "CM_GENERIC_PYTHON_PIP_URL": "" - } - }, - "url.#": { - "env": { - "CM_GENERIC_PYTHON_PIP_URL": "#", - "CM_TMP_PYTHON_PACKAGE_FORCE_INSTALL": "yes" - } - }, - "conda.#": { - "ad": { - "python-pip": { - "tags": "_conda.#" - }, - "python3": { - "tags": "_conda.#" - } - } - }, - "custom-python": { + "Pillow": { "env": { - "CM_TMP_USE_CUSTOM_PYTHON": "on" + "CM_GENERIC_PYTHON_PACKAGE_NAME": "Pillow" }, - "ad": { - "python-pip": { - "tags": "_custom-python" - } - } + "new_env_keys": [ + "CM_PILLOW_VERSION" + ] }, "apache-tvm": { "deps": [ @@ -108,6 +95,14 @@ "CM_APACHE_TVM_VERSION" ] }, + "apex": { + "env": { + "CM_GENERIC_PYTHON_PACKAGE_NAME": "apex" + }, + "new_env_keys": [ + "CM_APEX_VERSION" + ] + }, "async_timeout": { "env": { "CM_GENERIC_PYTHON_PACKAGE_NAME": "async_timeout" @@ -116,6 +111,14 @@ "CM_ASYNC_TIMEOUT_VERSION" ] }, + "attr": { + "env": { + "CM_GENERIC_PYTHON_PACKAGE_NAME": "attr" + }, + "new_env_keys": [ + "CM_ATTR_VERSION" + ] + }, "attrs": { "env": { "CM_GENERIC_PYTHON_PACKAGE_NAME": "attrs" @@ -124,93 +127,74 @@ "CM_ATTRS_VERSION" ] }, - "apex": { + "boto3": { "env": { - "CM_GENERIC_PYTHON_PACKAGE_NAME": "apex" + "CM_GENERIC_PYTHON_PACKAGE_NAME": "boto3" }, "new_env_keys": [ - "CM_APEX_VERSION" + "CM_BOTO3_VERSION" ] }, - "safetensors": { + "cloudpickle": { "env": { - "CM_GENERIC_PYTHON_PACKAGE_NAME": "safetensors" + "CM_GENERIC_PYTHON_PACKAGE_NAME": "cloudpickle" }, "new_env_keys": [ - "CM_SAFETENSORS_VERSION" - ], - "deps": [ - { - "tags": "get,rust-compiler", - "skip_if_env": { - "CM_HOST_PLATFORM_FLAVOR": [ "x86_64" ] - } - } + "CM_CLOUDPICKLE_VERSION" ] }, - "nvidia-apex": { + "cmind": { "env": { - "CM_GENERIC_PYTHON_PACKAGE_NAME": "apex", - "CM_GENERIC_PYTHON_PACKAGE_VARIANT": "nvidia-apex", - "CM_GENERIC_PYTHON_PIP_URL": "git+https://github.com/nvidia/apex@0da3ffb92ee6fbe5336602f0e3989db1cd16f880" + "CM_GENERIC_PYTHON_PACKAGE_NAME": "cmind" }, "new_env_keys": [ - "CM_NVIDIA_APEX_VERSION" + "CM_CMIND_VERSION" ] }, - "nvidia-apex-from-src": { + "colored": { "env": { - "CM_GENERIC_PYTHON_PACKAGE_NAME": "apex", - "CM_GENERIC_PYTHON_PACKAGE_VARIANT": "nvidia-apex" + "CM_GENERIC_PYTHON_PACKAGE_NAME": "colored", + "CM_GENERIC_PYTHON_PIP_EXTRA_INDEX_URL": "https://pypi.ngc.nvidia.com" }, "new_env_keys": [ - "CM_NVIDIA_APEX_VERSION" - ], + "CM_COLORED_VERSION" + ] + }, + "conda.#": { + "ad": { + "python-pip": { + "tags": "_conda.#" + }, + "python3": { + "tags": "_conda.#" + } + } + }, + "cupy": { "deps": [ { "names": [ "cuda" ], "tags": "get,cuda" - }, - { - "names": [ - "torch" - ], - "tags": "get,generic-python-lib,_torch_cuda" - }, - { - "tags": "get,git,repo,_repo.https://github.com/NVIDIA/apex,_tag.23.05", - "env": { - "CM_GIT_CHECKOUT_FOLDER": "apex" - }, - "extra_cache_tags": "nvidia-apex" } - ] - }, - "google-api-python-client": { + ], "env": { - "CM_GENERIC_PYTHON_PACKAGE_NAME": "google_api_python_client" + "CM_GENERIC_PYTHON_PACKAGE_NAME": "cupy" }, "new_env_keys": [ - "CM_GOOGLE_API_PYTHON_CLIENT_VERSION" + "CM_CUPY_VERSION" ] }, - "scons": { - "env": { - "CM_GENERIC_PYTHON_PACKAGE_NAME": "scons" + "custom-python": { + "ad": { + "python-pip": { + "tags": "_custom-python" + } }, - "new_env_keys": [ - "CM_SCONS_VERSION" - ] - }, - "google-auth-oauthlib": { "env": { - "CM_GENERIC_PYTHON_PACKAGE_NAME": "google_auth_oauthlib" - }, - "new_env_keys": [ - "CM_GOOGLE_AUTH_OAUTHLIB_VERSION" - ] + "CM_TMP_USE_CUSTOM_PYTHON": "on" + } }, "datasets": { "env": { @@ -220,29 +204,27 @@ "CM_DATASETS_VERSION" ] }, - "tensorboard": { + "decorator": { "env": { - "CM_GENERIC_PYTHON_PACKAGE_NAME": "tensorboard" + "CM_GENERIC_PYTHON_PACKAGE_NAME": "decorator" }, "new_env_keys": [ - "CM_TENSORBOARD_VERSION" + "CM_DECORATOR_VERSION" ] }, - "Pillow": { + "deepsparse": { "env": { - "CM_GENERIC_PYTHON_PACKAGE_NAME": "Pillow" + "CM_GENERIC_PYTHON_PACKAGE_NAME": "deepsparse" }, "new_env_keys": [ - "CM_PILLOW_VERSION" + "CM_DEEPSPARSE_VERSION" ] }, - "boto3": { + "dllogger": { "env": { - "CM_GENERIC_PYTHON_PACKAGE_NAME": "boto3" - }, - "new_env_keys": [ - "CM_BOTO3_VERSION" - ] + "CM_GENERIC_PYTHON_PACKAGE_NAME": "dllogger", + "CM_GENERIC_PYTHON_PIP_URL": "git+https://github.com/NVIDIA/dllogger#egg=dllogger" + } }, "fiftyone": { "env": { @@ -252,45 +234,28 @@ "CM_FIFTYONE_VERSION" ] }, - "cmind": { - "env": { - "CM_GENERIC_PYTHON_PACKAGE_NAME": "cmind" - }, - "new_env_keys": [ - "CM_CMIND_VERSION" - ] - }, - "colored": { - "env": { - "CM_GENERIC_PYTHON_PACKAGE_NAME": "colored", - "CM_GENERIC_PYTHON_PIP_EXTRA_INDEX_URL": "https://pypi.ngc.nvidia.com" - }, - "new_env_keys": [ - "CM_COLORED_VERSION" - ] - }, - "attr": { + "google-api-python-client": { "env": { - "CM_GENERIC_PYTHON_PACKAGE_NAME": "attr" + "CM_GENERIC_PYTHON_PACKAGE_NAME": "google_api_python_client" }, "new_env_keys": [ - "CM_ATTR_VERSION" + "CM_GOOGLE_API_PYTHON_CLIENT_VERSION" ] }, - "decorator": { + "google-auth-oauthlib": { "env": { - "CM_GENERIC_PYTHON_PACKAGE_NAME": "decorator" + "CM_GENERIC_PYTHON_PACKAGE_NAME": "google_auth_oauthlib" }, "new_env_keys": [ - "CM_DECORATOR_VERSION" + "CM_GOOGLE_AUTH_OAUTHLIB_VERSION" ] }, - "deepsparse": { + "huggingface_hub": { "env": { - "CM_GENERIC_PYTHON_PACKAGE_NAME": "deepsparse" + "CM_GENERIC_PYTHON_PACKAGE_NAME": "huggingface_hub" }, "new_env_keys": [ - "CM_DEEPSPARSE_VERSION" + "CM_HUGGINGFACE_HUB_VERSION" ] }, "inflect": { @@ -361,12 +326,6 @@ "CM_MLPERF_LOGGING_VERSION" ] }, - "dllogger": { - "env": { - "CM_GENERIC_PYTHON_PACKAGE_NAME": "dllogger", - "CM_GENERIC_PYTHON_PIP_URL": "git+https://github.com/NVIDIA/dllogger#egg=dllogger" - } - }, "mpld3": { "env": { "CM_GENERIC_PYTHON_PACKAGE_NAME": "mpld3" @@ -391,6 +350,46 @@ "CM_NUMPY_VERSION" ] }, + "nvidia-apex": { + "env": { + "CM_GENERIC_PYTHON_PACKAGE_NAME": "apex", + "CM_GENERIC_PYTHON_PACKAGE_VARIANT": "nvidia-apex", + "CM_GENERIC_PYTHON_PIP_URL": "git+https://github.com/nvidia/apex@0da3ffb92ee6fbe5336602f0e3989db1cd16f880" + }, + "new_env_keys": [ + "CM_NVIDIA_APEX_VERSION" + ] + }, + "nvidia-apex-from-src": { + "deps": [ + { + "names": [ + "cuda" + ], + "tags": "get,cuda" + }, + { + "names": [ + "torch" + ], + "tags": "get,generic-python-lib,_torch_cuda" + }, + { + "env": { + "CM_GIT_CHECKOUT_FOLDER": "apex" + }, + "extra_cache_tags": "nvidia-apex", + "tags": "get,git,repo,_repo.https://github.com/NVIDIA/apex,_tag.23.05" + } + ], + "env": { + "CM_GENERIC_PYTHON_PACKAGE_NAME": "apex", + "CM_GENERIC_PYTHON_PACKAGE_VARIANT": "nvidia-apex" + }, + "new_env_keys": [ + "CM_NVIDIA_APEX_VERSION" + ] + }, "nvidia-dali": { "deps": [ { @@ -413,10 +412,6 @@ "base": [ "pycocotools" ], - "env": { - "CM_GENERIC_PYTHON_PIP_UNINSTALL_DEPS": "pycocotools", - "CM_GENERIC_PYTHON_PIP_URL": "pycocotools@git+https://github.com/NVIDIA/cocoapi#subdirectory=PythonAPI" - }, "deps": [ { "names": [ @@ -430,7 +425,11 @@ ], "tags": "get,generic-python-lib,_package.numpy" } - ] + ], + "env": { + "CM_GENERIC_PYTHON_PIP_UNINSTALL_DEPS": "pycocotools", + "CM_GENERIC_PYTHON_PIP_URL": "pycocotools@git+https://github.com/NVIDIA/cocoapi#subdirectory=PythonAPI" + } }, "nvidia-pyindex": { "env": { @@ -456,12 +455,17 @@ "CM_ONNX_VERSION" ] }, - "huggingface_hub": { + "onnx-graphsurgeon": { + "deps": [ + { + "tags": "get,generic-python-lib,_package.nvidia-pyindex" + } + ], "env": { - "CM_GENERIC_PYTHON_PACKAGE_NAME": "huggingface_hub" + "CM_GENERIC_PYTHON_PACKAGE_NAME": "onnx_graphsurgeon" }, "new_env_keys": [ - "CM_HUGGINGFACE_HUB_VERSION" + "CM_ONNX_GRAPHSURGEON_VERSION" ] }, "onnxruntime": { @@ -472,7 +476,19 @@ "CM_ONNXRUNTIME_VERSION" ] }, + "onnxruntime,rocm": { + "env": { + "CM_GENERIC_PYTHON_PACKAGE_NAME": "onnxruntime-training", + "CM_GENERIC_PYTHON_PIP_URL": "https://download.onnxruntime.ai/onnxruntime_training-1.16.0%2Brocm56-cp3<<>>-cp3<<>>-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" + }, + "new_env_keys": [ + "CM_ONNXRUNTIME_TRAINING_VERSION*" + ] + }, "onnxruntime_gpu": { + "default_env": { + "CM_GENERIC_PYTHON_PIP_UNINSTALL_DEPS": "onnxruntime" + }, "deps": [ { "names": [ @@ -481,9 +497,6 @@ "tags": "get,cuda" } ], - "default_env": { - "CM_GENERIC_PYTHON_PIP_UNINSTALL_DEPS": "onnxruntime" - }, "env": { "CM_GENERIC_PYTHON_PACKAGE_NAME": "onnxruntime_gpu", "CM_ONNXRUNTIME_VERSION_EXTRA": "GPU" @@ -492,15 +505,6 @@ "CM_ONNXRUNTIME_GPU_VERSION*" ] }, - "onnxruntime,rocm": { - "env": { - "CM_GENERIC_PYTHON_PACKAGE_NAME": "onnxruntime-training", - "CM_GENERIC_PYTHON_PIP_URL": "https://download.onnxruntime.ai/onnxruntime_training-1.16.0%2Brocm56-cp3<<>>-cp3<<>>-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" - }, - "new_env_keys": [ - "CM_ONNXRUNTIME_TRAINING_VERSION*" - ] - }, "opencv-python": { "env": { "CM_GENERIC_PYTHON_PACKAGE_NAME": "opencv-python" @@ -509,6 +513,13 @@ "CM_OPENCV_PYTHON_VERSION" ] }, + "package.#": { + "env": { + "CM_GENERIC_PYTHON_PACKAGE_NAME": "#", + "CM_GENERIC_PYTHON_PIP_UNINSTALL_DEPS": "", + "CM_GENERIC_PYTHON_PIP_URL": "" + } + }, "pandas": { "env": { "CM_GENERIC_PYTHON_PACKAGE_NAME": "pandas" @@ -517,6 +528,11 @@ "CM_PANDAS_VERSION" ] }, + "path.#": { + "env": { + "CM_GENERIC_PYTHON_PIP_URL": "#" + } + }, "pillow": { "env": { "CM_GENERIC_PYTHON_PACKAGE_NAME": "Pillow" @@ -525,22 +541,6 @@ "CM_PILLOW_VERSION" ] }, - "sparsezoo": { - "env": { - "CM_GENERIC_PYTHON_PACKAGE_NAME": "sparsezoo" - }, - "new_env_keys": [ - "CM_SPARSEZOO_VERSION" - ] - }, - "setfit": { - "env": { - "CM_GENERIC_PYTHON_PACKAGE_NAME": "setfit" - }, - "new_env_keys": [ - "CM_SETFIT_VERSION" - ] - }, "pip": { "env": { "CM_GENERIC_PYTHON_PACKAGE_NAME": "pip" @@ -564,18 +564,10 @@ "CM_POLYGRAPHY_VERSION" ] }, - "onnx-graphsurgeon": { - "deps": [ - { - "tags": "get,generic-python-lib,_package.nvidia-pyindex" - } - ], + "pre": { "env": { - "CM_GENERIC_PYTHON_PACKAGE_NAME": "onnx_graphsurgeon" - }, - "new_env_keys": [ - "CM_ONNX_GRAPHSURGEON_VERSION" - ] + "CM_GENERIC_PYTHON_DEV_VERSION": "yes" + } }, "protobuf": { "env": { @@ -633,6 +625,35 @@ "CM_REQUESTS_VERSION" ] }, + "rocm": { + "deps": [ + { + "names": [ + "rocm" + ], + "tags": "get,rocm" + } + ], + "env": {} + }, + "safetensors": { + "deps": [ + { + "skip_if_env": { + "CM_HOST_PLATFORM_FLAVOR": [ + "x86_64" + ] + }, + "tags": "get,rust-compiler" + } + ], + "env": { + "CM_GENERIC_PYTHON_PACKAGE_NAME": "safetensors" + }, + "new_env_keys": [ + "CM_SAFETENSORS_VERSION" + ] + }, "scikit-learn": { "env": { "CM_GENERIC_PYTHON_PACKAGE_NAME": "scikit-learn" @@ -641,28 +662,28 @@ "CM_SCIKIT_LEARN_VERSION" ] }, - "cupy": { - "deps": [ - { - "names": [ - "cuda" - ], - "tags": "get,cuda" - } - ], + "scipy": { "env": { - "CM_GENERIC_PYTHON_PACKAGE_NAME": "cupy" + "CM_GENERIC_PYTHON_PACKAGE_NAME": "scipy" }, "new_env_keys": [ - "CM_CUPY_VERSION" + "CM_SCIPY_VERSION" ] }, - "scipy": { + "scons": { + "env": { + "CM_GENERIC_PYTHON_PACKAGE_NAME": "scons" + }, + "new_env_keys": [ + "CM_SCONS_VERSION" + ] + }, + "setfit": { "env": { - "CM_GENERIC_PYTHON_PACKAGE_NAME": "scipy" + "CM_GENERIC_PYTHON_PACKAGE_NAME": "setfit" }, "new_env_keys": [ - "CM_SCIPY_VERSION" + "CM_SETFIT_VERSION" ] }, "setuptools": { @@ -673,6 +694,14 @@ "CM_SETUPTOOL_VERSION" ] }, + "six": { + "env": { + "CM_GENERIC_PYTHON_PACKAGE_NAME": "six" + }, + "new_env_keys": [ + "CM_SIX_VERSION" + ] + }, "sklearn": { "env": { "CM_GENERIC_PYTHON_PACKAGE_NAME": "sklearn" @@ -689,6 +718,14 @@ "CM_SOX_VERSION" ] }, + "sparsezoo": { + "env": { + "CM_GENERIC_PYTHON_PACKAGE_NAME": "sparsezoo" + }, + "new_env_keys": [ + "CM_SPARSEZOO_VERSION" + ] + }, "streamlit": { "env": { "CM_GENERIC_PYTHON_PACKAGE_NAME": "streamlit" @@ -705,6 +742,14 @@ "CM_STREAMLIT_OPTION_MENU_VERSION" ] }, + "tensorboard": { + "env": { + "CM_GENERIC_PYTHON_PACKAGE_NAME": "tensorboard" + }, + "new_env_keys": [ + "CM_TENSORBOARD_VERSION" + ] + }, "tensorflow": { "env": { "CM_GENERIC_PYTHON_PACKAGE_NAME": "tensorflow" @@ -721,22 +766,6 @@ "CM_TENSORFLOW_ROCM_VERSION" ] }, - "tflite-runtime": { - "env": { - "CM_GENERIC_PYTHON_PACKAGE_NAME": "tflite-runtime" - }, - "new_env_keys": [ - "CM_TFLITE_RUNTIME_VERSION" - ] - }, - "tokenization": { - "env": { - "CM_GENERIC_PYTHON_PACKAGE_NAME": "tokenization" - }, - "new_env_keys": [ - "CM_TOKENIZATION_VERSION" - ] - }, "tensorrt": { "env": { "CM_GENERIC_PYTHON_PACKAGE_NAME": "tensorrt", @@ -747,73 +776,36 @@ "CM_TENSORRT_VERSION" ] }, - "torch_tensorrt": { - "env": { - "CM_GENERIC_PYTHON_PACKAGE_NAME": "torch-tensorrt", - "CM_GENERIC_PYTHON_PIP_EXTRA_INDEX_URL": "https://download.pytorch.org/whl/<<>>", - "CM_TORCH_VERSION_EXTRA": "CUDA" - }, - "new_env_keys": [ - "CM_TORCH_TENSORRT_VERSION" - ] - }, - "six": { + "tflite": { "env": { - "CM_GENERIC_PYTHON_PACKAGE_NAME": "six" + "CM_GENERIC_PYTHON_PACKAGE_NAME": "tflite" }, "new_env_keys": [ - "CM_SIX_VERSION" + "CM_TFLITE_VERSION" ] }, - "toml": { + "tflite-runtime": { "env": { - "CM_GENERIC_PYTHON_PACKAGE_NAME": "toml" + "CM_GENERIC_PYTHON_PACKAGE_NAME": "tflite-runtime" }, "new_env_keys": [ - "CM_TOML_VERSION" + "CM_TFLITE_RUNTIME_VERSION" ] }, - "pre": { - "env": { - "CM_GENERIC_PYTHON_DEV_VERSION": "yes" - } - }, - "torch,pre": { - "default_env": { - "CM_GENERIC_PYTHON_PIP_UNINSTALL_DEPS": "torch" - }, + "tokenization": { "env": { - "CM_GENERIC_PYTHON_PACKAGE_NAME": "torch", - "CM_GENERIC_PYTHON_PIP_EXTRA": " --pre", - "CM_GENERIC_PYTHON_PIP_INDEX_URL": "https://download.pytorch.org/whl/nightly/cpu" + "CM_GENERIC_PYTHON_PACKAGE_NAME": "tokenization" }, "new_env_keys": [ - "CM_TORCH_VERSION*" + "CM_TOKENIZATION_VERSION" ] }, - "torch_cuda,pre": { - "deps": [ - { - "names": [ - "cuda" - ], - "tags": "get,cuda" - }, - { - "tags": "get,generic-python-lib,_numpy" - } - ], - "default_env": { - "CM_GENERIC_PYTHON_PIP_UNINSTALL_DEPS": "torch_cuda" - }, + "toml": { "env": { - "CM_GENERIC_PYTHON_PACKAGE_NAME": "torch", - "CM_GENERIC_PYTHON_PIP_INDEX_URL": "https://download.pytorch.org/whl/<<>>", - "CM_GENERIC_PYTHON_PIP_EXTRA": " --pre", - "CM_TORCH_VERSION_EXTRA": "CUDA" + "CM_GENERIC_PYTHON_PACKAGE_NAME": "toml" }, "new_env_keys": [ - "CM_TORCH_VERSION*" + "CM_TOML_VERSION" ] }, "torch": { @@ -825,40 +817,23 @@ "CM_TORCH_VERSION*" ] }, - "rocm": { - "env": { - }, - "deps": [ - { - "names": [ "rocm" ], - "tags": "get,rocm" - } - ] - }, - "torchvision,rocm": { - "env": { - "CM_GENERIC_PYTHON_PIP_INDEX_URL": "https://download.pytorch.org/whl/rocm5.6", - "CM_GENERIC_PYTHON_PACKAGE_NAME": "torchvision", - "CM_GENERIC_PYTHON_PIP_UNINSTALL_DEPS": "torchvision" + "torch,pre": { + "default_env": { + "CM_GENERIC_PYTHON_PIP_UNINSTALL_DEPS": "torch" }, - "new_env_keys": [ - "CM_TORCHVISION_VERSION*" - ] - }, - "torchaudio,rocm": { "env": { - "CM_GENERIC_PYTHON_PIP_INDEX_URL": "https://download.pytorch.org/whl/rocm5.6", - "CM_GENERIC_PYTHON_PACKAGE_NAME": "torchaudio", - "CM_GENERIC_PYTHON_PIP_UNINSTALL_DEPS": "torchaudio" + "CM_GENERIC_PYTHON_PACKAGE_NAME": "torch", + "CM_GENERIC_PYTHON_PIP_EXTRA": " --pre", + "CM_GENERIC_PYTHON_PIP_INDEX_URL": "https://download.pytorch.org/whl/nightly/cpu" }, "new_env_keys": [ - "CM_TORCHAUDIO_VERSION*" + "CM_TORCH_VERSION*" ] }, "torch,rocm": { "env": { - "CM_GENERIC_PYTHON_PIP_INDEX_URL": "https://download.pytorch.org/whl/rocm5.6", "CM_GENERIC_PYTHON_PACKAGE_NAME": "torch", + "CM_GENERIC_PYTHON_PIP_INDEX_URL": "https://download.pytorch.org/whl/rocm5.6", "CM_GENERIC_PYTHON_PIP_UNINSTALL_DEPS": "torch" }, "new_env_keys": [ @@ -874,6 +849,9 @@ ] }, "torch_cuda": { + "default_env": { + "CM_GENERIC_PYTHON_PIP_UNINSTALL_DEPS1": "torch" + }, "deps": [ { "names": [ @@ -882,18 +860,50 @@ "tags": "get,cuda" } ], + "env": { + "CM_GENERIC_PYTHON_PACKAGE_NAME": "torch", + "CM_GENERIC_PYTHON_PIP_EXTRA_INDEX_URL1": "https://download.pytorch.org/whl/<<>>", + "CM_TORCH_VERSION_EXTRA": "CUDA" + }, + "new_env_keys": [ + "CM_TORCH_VERSION*" + ] + }, + "torch_cuda,pre": { "default_env": { - "CM_GENERIC_PYTHON_PIP_UNINSTALL_DEPS1": "torch" + "CM_GENERIC_PYTHON_PIP_UNINSTALL_DEPS": "torch_cuda" }, + "deps": [ + { + "names": [ + "cuda" + ], + "tags": "get,cuda" + }, + { + "tags": "get,generic-python-lib,_numpy" + } + ], "env": { "CM_GENERIC_PYTHON_PACKAGE_NAME": "torch", - "CM_GENERIC_PYTHON_PIP_EXTRA_INDEX_URL1": "https://download.pytorch.org/whl/<<>>", + "CM_GENERIC_PYTHON_PIP_EXTRA": " --pre", + "CM_GENERIC_PYTHON_PIP_INDEX_URL": "https://download.pytorch.org/whl/<<>>", "CM_TORCH_VERSION_EXTRA": "CUDA" }, "new_env_keys": [ "CM_TORCH_VERSION*" ] }, + "torch_tensorrt": { + "env": { + "CM_GENERIC_PYTHON_PACKAGE_NAME": "torch-tensorrt", + "CM_GENERIC_PYTHON_PIP_EXTRA_INDEX_URL": "https://download.pytorch.org/whl/<<>>", + "CM_TORCH_VERSION_EXTRA": "CUDA" + }, + "new_env_keys": [ + "CM_TORCH_TENSORRT_VERSION" + ] + }, "torchaudio": { "env": { "CM_GENERIC_PYTHON_PACKAGE_NAME": "torchaudio", @@ -903,7 +913,20 @@ "CM_TORCHAUDIO_VERSION*" ] }, + "torchaudio,rocm": { + "env": { + "CM_GENERIC_PYTHON_PACKAGE_NAME": "torchaudio", + "CM_GENERIC_PYTHON_PIP_INDEX_URL": "https://download.pytorch.org/whl/rocm5.6", + "CM_GENERIC_PYTHON_PIP_UNINSTALL_DEPS": "torchaudio" + }, + "new_env_keys": [ + "CM_TORCHAUDIO_VERSION*" + ] + }, "torchaudio_cuda": { + "default_env": { + "CM_GENERIC_PYTHON_PIP_UNINSTALL_DEPS": "torchaudio" + }, "deps": [ { "names": [ @@ -912,9 +935,6 @@ "tags": "get,cuda" } ], - "default_env": { - "CM_GENERIC_PYTHON_PIP_UNINSTALL_DEPS": "torchaudio" - }, "env": { "CM_GENERIC_PYTHON_PACKAGE_NAME": "torchaudio", "CM_GENERIC_PYTHON_PIP_EXTRA_INDEX_URL1": "https://download.pytorch.org/whl/<<>>", @@ -933,7 +953,20 @@ "CM_TORCHVISION_VERSION*" ] }, + "torchvision,rocm": { + "env": { + "CM_GENERIC_PYTHON_PACKAGE_NAME": "torchvision", + "CM_GENERIC_PYTHON_PIP_INDEX_URL": "https://download.pytorch.org/whl/rocm5.6", + "CM_GENERIC_PYTHON_PIP_UNINSTALL_DEPS": "torchvision" + }, + "new_env_keys": [ + "CM_TORCHVISION_VERSION*" + ] + }, "torchvision_cuda": { + "default_env": { + "CM_GENERIC_PYTHON_PIP_UNINSTALL_DEPS": "torchvision" + }, "deps": [ { "names": [ @@ -942,9 +975,6 @@ "tags": "get,cuda" } ], - "default_env": { - "CM_GENERIC_PYTHON_PIP_UNINSTALL_DEPS": "torchvision" - }, "env": { "CM_GENERIC_PYTHON_PACKAGE_NAME": "torchvision", "CM_TORCHVISION_VERSION_EXTRA": "CUDA" @@ -953,6 +983,14 @@ "CM_TORCHVISION_VERSION*" ] }, + "tornado": { + "env": { + "CM_GENERIC_PYTHON_PACKAGE_NAME": "tornado" + }, + "new_env_keys": [ + "CM_TORNADO_VERSION" + ] + }, "tqdm": { "env": { "CM_GENERIC_PYTHON_PACKAGE_NAME": "tqdm" @@ -993,6 +1031,12 @@ "CM_UNIDECODE_VERSION" ] }, + "url.#": { + "env": { + "CM_GENERIC_PYTHON_PIP_URL": "#", + "CM_TMP_PYTHON_PACKAGE_FORCE_INSTALL": "yes" + } + }, "wandb": { "env": { "CM_GENERIC_PYTHON_PACKAGE_NAME": "wandb" @@ -1009,22 +1053,6 @@ "CM_WEST_VERSION" ] }, - "tflite": { - "env": { - "CM_GENERIC_PYTHON_PACKAGE_NAME": "tflite" - }, - "new_env_keys": [ - "CM_TFLITE_VERSION" - ] - }, - "xlsxwriter": { - "env": { - "CM_GENERIC_PYTHON_PACKAGE_NAME": "xlsxwriter" - }, - "new_env_keys": [ - "CM_XLSXWRITER_VERSION" - ] - }, "xgboost": { "env": { "CM_GENERIC_PYTHON_PACKAGE_NAME": "xgboost" @@ -1033,26 +1061,13 @@ "CM_XGBOOST_VERSION" ] }, - "cloudpickle": { - "env": { - "CM_GENERIC_PYTHON_PACKAGE_NAME": "cloudpickle" - }, - "new_env_keys": [ - "CM_CLOUDPICKLE_VERSION" - ] - }, - "tornado": { + "xlsxwriter": { "env": { - "CM_GENERIC_PYTHON_PACKAGE_NAME": "tornado" + "CM_GENERIC_PYTHON_PACKAGE_NAME": "xlsxwriter" }, "new_env_keys": [ - "CM_TORNADO_VERSION" + "CM_XLSXWRITER_VERSION" ] - }, - "path.#": { - "env": { - "CM_GENERIC_PYTHON_PIP_URL": "#" - } } } } diff --git a/cm-mlops/script/get-generic-python-lib/customize.py b/cm-mlops/script/get-generic-python-lib/customize.py index a7738beab..a6e9590f6 100644 --- a/cm-mlops/script/get-generic-python-lib/customize.py +++ b/cm-mlops/script/get-generic-python-lib/customize.py @@ -47,7 +47,7 @@ def preprocess(i): 'run_script_input':i['run_script_input'], 'recursion_spaces':recursion_spaces}) - force_install = env.get('CM_TMP_PYTHON_PACKAGE_FORCE_INSTALL', '') == 'yes' + force_install = (env.get('CM_TMP_PYTHON_PACKAGE_FORCE_INSTALL', '') in ['yes', 'true', 'True', True]) if r['return'] >0 or force_install: if r['return'] == 16 or force_install: @@ -57,6 +57,10 @@ def preprocess(i): if env.get(env_version_key,'')!='': del(env[env_version_key]) + # Check if upgrade + if force_install: + extra+=' --upgrade --no-deps --force-reinstall' + # Check index URL index_url = env.get('CM_GENERIC_PYTHON_PIP_INDEX_URL','').strip() if index_url != '': @@ -79,6 +83,10 @@ def preprocess(i): if env.get('CM_GENERIC_PYTHON_PIP_UPDATE','') in [True,'true','yes','on']: extra +=' -U' + print ('') + print (recursion_spaces + ' Extra PIP CMD: ' + extra) + print ('') + env['CM_GENERIC_PYTHON_PIP_EXTRA'] = extra package_name = env.get('CM_GENERIC_PYTHON_PACKAGE_NAME', '').strip() diff --git a/cm-mlops/script/get-generic-python-lib/install.bat b/cm-mlops/script/get-generic-python-lib/install.bat index 7dcd17c89..0a5967462 100644 --- a/cm-mlops/script/get-generic-python-lib/install.bat +++ b/cm-mlops/script/get-generic-python-lib/install.bat @@ -1,3 +1,5 @@ +echo. + if NOT "%CM_GENERIC_PYTHON_PIP_URL%" == "" ( %CM_PYTHON_BIN_WITH_PATH% -m pip install %CM_GENERIC_PYTHON_PIP_URL% %CM_GENERIC_PYTHON_PIP_EXTRA% diff --git a/cm-mlops/script/get-generic-python-lib/install.sh b/cm-mlops/script/get-generic-python-lib/install.sh index 517a4e728..6029190cc 100644 --- a/cm-mlops/script/get-generic-python-lib/install.sh +++ b/cm-mlops/script/get-generic-python-lib/install.sh @@ -1,5 +1,7 @@ #!/bin/bash +echo "" + if [[ ${CM_GENERIC_PYTHON_PACKAGE_VARIANT} == "nvidia-apex-depreciated" ]]; then cd ${CM_GIT_REPO_CHECKOUT_PATH} cmd="${CM_PYTHON_BIN_WITH_PATH} -m pip install -v --disable-pip-version-check --global-option=\"--cpp_ext\" --global-option=\"--cuda_ext\" ./" diff --git a/cm/CHANGES.md b/cm/CHANGES.md index 0dced6177..cd45f122b 100644 --- a/cm/CHANGES.md +++ b/cm/CHANGES.md @@ -1,8 +1,9 @@ -## V2.0.3.1 +## V2.0.4 - added skip of delayed help to simplify output of `cmr [tags] --help` - revisited automatically generated READMEs for CM scripts (automation recipes) based on user feedback: https://github.com/mlcommons/ck/issues/1169 - - improved deleting of CM entries on Windows + - improved deleting of CM artifacts (entries) on Windows + - print tags when deleting CM artifacts (entries) ## V2.0.3 - added support to handle broken CM repositories: https://github.com/mlcommons/ck/issues/1177 diff --git a/cm/cmind/__init__.py b/cm/cmind/__init__.py index be695ed86..d0ec81781 100644 --- a/cm/cmind/__init__.py +++ b/cm/cmind/__init__.py @@ -1,4 +1,4 @@ -__version__ = "2.0.3.1" +__version__ = "2.0.4" from cmind.core import access from cmind.core import error diff --git a/cm/cmind/automation.py b/cm/cmind/automation.py index 9e839074d..f38447ba3 100644 --- a/cm/cmind/automation.py +++ b/cm/cmind/automation.py @@ -576,7 +576,10 @@ def delete(self, i): path_to_artifact = artifact.path if console: - print ('Deleting CM artifact in {} ...'.format(path_to_artifact)) + tags = artifact.meta.get('tags',[]) + x = '' if len(tags)=='' else ' with tags "{}"'.format(','.join(tags)) + + print ('Deleting CM artifact in {}{} ...'.format(path_to_artifact, x)) if not force: ask = input(' Are you sure you want to delete this artifact (y/N): ') diff --git a/cmr.yaml b/cmr.yaml index b06dee420..7795aee19 100644 --- a/cmr.yaml +++ b/cmr.yaml @@ -5,4 +5,4 @@ git: true prefix: cm-mlops -version: 2.0.3 +version: 2.0.4