Merge from cTuning (#775)

cm-mlops/automation/script/module_misc.py

@@ -145,7 +145,7 @@ def doc(i):
 
     list_of_repos = repos.split(',') if ',' in repos else [repos]
 
-    ii = utils.sub_input(i, self_module.cmind.cfg['artifact_keys'])
+    ii = utils.sub_input(i, self_module.cmind.cfg['artifact_keys'] + ['tags'])
 
     ii['out'] = None
 
@@ -1085,7 +1085,7 @@ def dockerfile(i):
 
     list_of_repos = repos.split(',') if ',' in repos else [repos]
 
-    ii = utils.sub_input(i, self_module.cmind.cfg['artifact_keys'])
+    ii = utils.sub_input(i, self_module.cmind.cfg['artifact_keys'] + ['tags'])
 
     ii['out'] = None
 
@@ -1098,32 +1098,7 @@ def dockerfile(i):
         if r['return']>0: return r
         lst += r['list']
 
-    md = []
-
-    toc = []
-
-    script_meta = {}
-    urls = {}
-
-    if i.get("all_os"):
-        docker_os = {
-            "ubuntu": ["18.04","20.04","22.04"],
-            "rhel": ["9"]
-        }
-    else:
-        docker_os = {}
-        if i.get('docker_os'):
-            docker_os[i['docker_os']] = []
-        if i.get('docker_os_version'):
-            docker_os[i['docker_os']] = [i.get('docker_os_version')]
-        else:
-            if docker_os == "ubuntu":
-                docker_os["ubuntu"] = ["22.04"]
-
-        if not docker_os:
-          docker_os = {
-            "ubuntu": ["22.04"],
-          }
+    run_cmd = "cm run script " + " ".join(i['cmd'])
 
     for artifact in sorted(lst, key = lambda x: x.meta.get('alias','')):
 
@@ -1132,48 +1107,60 @@ def dockerfile(i):
         tags = meta.get("tags", [])
         tag_string=",".join(tags)
 
-        for _os in docker_os:
-            for version in docker_os[_os]:
-                dockerfile_path = os.path.join(script_path,'dockerfiles', _os +'_'+version +'.Dockerfile')
-                if i.get('print_deps'):
-                  cm_input = {'action': 'run',
-                            'automation': 'script',
-                            'tags': f'{tag_string}',
-                            'print_deps': True,
-                            'quiet': True,
-                            'silent': True,
-                            'fake_run': True
-                            }
-                  r = self_module.cmind.access(cm_input)
-                  if r['return'] > 0:
-                    return r
-                  print_deps = r['new_state']['print_deps']
-                  comments = [ "#RUN " + dep for dep in print_deps ]
-                  comments.append("")
-                  comments.append("# Run CM workflow")
-                else:
-                  comments = []
+        run_config_path = os.path.join(script_path,'run_config.yml')
+        if not os.path.exists(run_config_path):
+            print("No run_config.yml file present in {}".format(script_path))
+            continue
+        import yaml
+        with open(run_config_path, 'r') as run_config_file:
+            run_config = yaml.safe_load(run_config_file)
+        docker_settings = run_config.get('docker')
+        if not docker_settings or not docker_settings.get('build') or not run_config.get('run_with_default_inputs'):
+            print("Run config is not configured for docker run in {}".format(run_config_path))
+            continue
+        docker_os = docker_settings.get('docker_os', 'ubuntu')
+        docker_os_version = docker_settings.get('docker_os_version', '22.04')
+
+        dockerfile_path = os.path.join(script_path,'dockerfiles', docker_os +'_'+docker_os_version +'.Dockerfile')
+        if i.get('print_deps'):
+            cm_input = {'action': 'run',
+                    'automation': 'script',
+                    'tags': f'{tag_string}',
+                    'print_deps': True,
+                    'quiet': True,
+                    'silent': True,
+                    'fake_run': True
+                    }
+            r = self_module.cmind.access(cm_input)
+            if r['return'] > 0:
+                return r
+            print_deps = r['new_state']['print_deps']
+            comments = [ "#RUN " + dep for dep in print_deps ]
+            comments.append("")
+            comments.append("# Run CM workflow")
+        else:
+            comments = []
 
-                cm_docker_input = {'action': 'run',
+        cm_docker_input = {'action': 'run',
                             'automation': 'script',
                             'tags': 'build,dockerfile',
                             'cm_repo': cm_repo,
-                            'docker_os': _os,
-                            'docker_os_version': version,
+                            'docker_os': docker_os,
+                            'docker_os_version': docker_os_version,
                             'file_path': dockerfile_path,
                             'comments': comments,
-                            'run_cmd': f'cm run script --tags={tag_string} --quiet',
+                            'run_cmd': f'{run_cmd} --quiet',
                             'script_tags': f'{tag_string}',
                             'quiet': True,
                             'print_deps': True,
                             'real_run': True
                             }
 
-                r = self_module.cmind.access(cm_docker_input)
-                if r['return'] > 0:
-                    return r
+        r = self_module.cmind.access(cm_docker_input)
+        if r['return'] > 0:
+            return r
 
-                print("Dockerfile generated at "+dockerfile_path)
+        print("Dockerfile generated at "+dockerfile_path)
 
     return {'return':0}
 
@@ -1226,7 +1213,7 @@ def docker(i):
 
     list_of_repos = repos.split(',') if ',' in repos else [repos]
 
-    ii = utils.sub_input(i, self_module.cmind.cfg['artifact_keys'])
+    ii = utils.sub_input(i, self_module.cmind.cfg['artifact_keys'] + ['tags'])
 
     ii['out'] = None
 
@@ -1239,12 +1226,7 @@ def docker(i):
         if r['return']>0: return r
         lst += r['list']
 
-    md = []
-
-    toc = []
-
-    script_meta = {}
-    urls = {}
+    run_cmd = "cm run script " + " ".join(i['cmd'])
 
     for artifact in sorted(lst, key = lambda x: x.meta.get('alias','')):
 
@@ -1269,6 +1251,7 @@ def docker(i):
                             'docker_os_version': version,
                             'detached': 'no',
                             'script_tags': f'{tag_string}',
+                            'run_cmd': run_cmd,
                             'quiet': True,
                             'real_run': True,
                             'add_deps_recursive': {

cm-mlops/script/app-mlperf-inference/run_config.yml

@@ -0,0 +1,11 @@
+docker:
+  build: true
+  docker_os: ubuntu
+  docker_os_version: "22.04"
+
+run_with_default_inputs: true #if false the script won't run automatic tests
+variation_combinations:
+  - _bert,_nvidia-original:
+      minimum_system_requirements:
+        ram: 4 #in GB
+        disk_space: 6 #in GB

cm-mlops/script/get-git-repo/_cm.json

@@ -25,6 +25,19 @@
       "tags": "detect,os"
     }
   ],
+  "post_deps": [
+    {
+      "tags": "pull,git,repo",
+      "names": [ "pull-git-repo" ],
+      "dynamic": true,
+      "enable_if_env": {
+        "CM_GIT_REPO_PULL": [ "yes", "True" ]
+      },
+      "force_env_keys": [
+        "CM_GIT_CHECKOUT_PATH"
+      ]
+    }
+  ],
   "new_env_keys": [
     "CM_GIT_CHECKOUT_PATH",
     "CM_GIT_REPO_*"

cm-mlops/script/pull-git-repo/README.md

@@ -0,0 +1,154 @@
+<details>
+<summary>Click here to see the table of contents.</summary>
+
+* [Description](#description)
+* [Information](#information)
+* [Usage](#usage)
+  * [ CM installation](#cm-installation)
+  * [ CM script automation help](#cm-script-automation-help)
+  * [ CM CLI](#cm-cli)
+  * [ CM Python API](#cm-python-api)
+  * [ CM GUI](#cm-gui)
+  * [ CM modular Docker container](#cm-modular-docker-container)
+* [Customization](#customization)
+  * [ Script flags mapped to environment](#script-flags-mapped-to-environment)
+  * [ Default environment](#default-environment)
+* [Script workflow, dependencies and native scripts](#script-workflow-dependencies-and-native-scripts)
+* [Script output](#script-output)
+* [New environment keys (filter)](#new-environment-keys-(filter))
+* [New environment keys auto-detected from customize](#new-environment-keys-auto-detected-from-customize)
+* [Maintainers](#maintainers)
+
+</details>
+
+*Note that this README is automatically generated - don't edit! Use `README-extra.md` to add more info.*
+
+### Description
+
+#### Information
+
+* Category: *Modular MLPerf benchmarks.*
+* CM GitHub repository: *[mlcommons@ck](https://github.com/mlcommons/ck/tree/master/cm-mlops)*
+* GitHub directory for this script: *[GitHub](https://github.com/mlcommons/ck/tree/master/cm-mlops/script/pull-git-repo)*
+* CM meta description for this script: *[_cm.json](_cm.json)*
+* CM "database" tags to find this script: *pull,git,repo,repository*
+* Output cached?: *False*
+___
+### Usage
+
+#### CM installation
+
+[Guide](https://github.com/mlcommons/ck/blob/master/docs/installation.md)
+
+##### CM pull repository
+
+```cm pull repo mlcommons@ck```
+
+##### CM script automation help
+
+```cm run script --help```
+
+#### CM CLI
+
+1. `cm run script --tags=pull,git,repo,repository [--input_flags]`
+
+2. `cm run script "pull git repo repository" [--input_flags]`
+
+3. `cm run script c23132ed65c4421d [--input_flags]`
+
+* `variations` can be seen [here](#variations)
+
+* `input_flags` can be seen [here](#script-flags-mapped-to-environment)
+
+#### CM Python API
+
+<details>
+<summary>Click here to expand this section.</summary>
+
+```python
+
+import cmind
+
+r = cmind.access({'action':'run'
+                  'automation':'script',
+                  'tags':'pull,git,repo,repository'
+                  'out':'con',
+                  ...
+                  (other input keys for this script)
+                  ...
+                 })
+
+if r['return']>0:
+    print (r['error'])
+
+```
+
+</details>
+
+
+#### CM GUI
+
+```cm run script --tags=gui --script="pull,git,repo,repository"```
+
+Use this [online GUI](https://cKnowledge.org/cm-gui/?tags=pull,git,repo,repository) to generate CM CMD.
+
+#### CM modular Docker container
+
+*TBD*
+
+___
+### Customization
+
+
+#### Script flags mapped to environment
+<details>
+<summary>Click here to expand this section.</summary>
+
+* `--path=value`  &rarr;  `CM_GIT_CHECKOUT_PATH=value`
+
+**Above CLI flags can be used in the Python CM API as follows:**
+
+```python
+r=cm.access({... , "path":...}
+```
+
+</details>
+
+#### Default environment
+
+<details>
+<summary>Click here to expand this section.</summary>
+
+These keys can be updated via `--env.KEY=VALUE` or `env` dictionary in `@input.json` or using script flags.
+
+
+</details>
+
+___
+### Script workflow, dependencies and native scripts
+
+<details>
+<summary>Click here to expand this section.</summary>
+
+  1. ***Read "deps" on other CM scripts from [meta](https://github.com/mlcommons/ck/tree/master/cm-mlops/script/pull-git-repo/_cm.json)***
+     * detect,os
+       - CM script: [detect-os](https://github.com/mlcommons/ck/tree/master/cm-mlops/script/detect-os)
+  1. ***Run "preprocess" function from [customize.py](https://github.com/mlcommons/ck/tree/master/cm-mlops/script/pull-git-repo/customize.py)***
+  1. Read "prehook_deps" on other CM scripts from [meta](https://github.com/mlcommons/ck/tree/master/cm-mlops/script/pull-git-repo/_cm.json)
+  1. ***Run native script if exists***
+     * [run.sh](https://github.com/mlcommons/ck/tree/master/cm-mlops/script/pull-git-repo/run.sh)
+  1. Read "posthook_deps" on other CM scripts from [meta](https://github.com/mlcommons/ck/tree/master/cm-mlops/script/pull-git-repo/_cm.json)
+  1. ***Run "postrocess" function from [customize.py](https://github.com/mlcommons/ck/tree/master/cm-mlops/script/pull-git-repo/customize.py)***
+  1. Read "post_deps" on other CM scripts from [meta](https://github.com/mlcommons/ck/tree/master/cm-mlops/script/pull-git-repo/_cm.json)
+</details>
+
+___
+### Script output
+#### New environment keys (filter)
+
+#### New environment keys auto-detected from customize
+
+___
+### Maintainers
+
+* [Open MLCommons taskforce on automation and reproducibility](https://github.com/mlcommons/ck/blob/master/docs/taskforce.md)

cm-mlops/script/pull-git-repo/_cm.json

@@ -0,0 +1,27 @@
+{
+  "alias": "pull-git-repo",
+  "automation_alias": "script",
+  "automation_uid": "5b4e0237da074764",
+  "category": "Modular MLPerf benchmarks",
+  "default_env": {
+  },
+  "deps": [
+    {
+      "tags": "detect,os"
+    }
+  ],
+  "input_mapping": {
+    "path": "CM_GIT_CHECKOUT_PATH"
+  },
+  "new_env_keys": [
+  ],
+  "tags": [
+    "pull",
+    "git",
+    "repo",
+    "repository"
+  ],
+  "uid": "c23132ed65c4421d",
+  "variations": {
+  }
+}