CM v2.3.5 release (#1280)

.github/workflows/publish.yaml

@@ -0,0 +1,33 @@
+# This is a basic workflow to help you get started with Actions
+
+name: Publish site
+
+
+on:
+  release:
+    types: [published]
+  push:
+    branches:
+      - main
+      - docs
+
+jobs:
+
+  publish:
+    name: Publish the site
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository normally
+        uses: actions/checkout@v3
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.11"
+
+      - name: Install Mkdocs
+        run: pip install -r docs/requirements.txt
+
+      - name: Run Mkdocs deploy
+        run: mkdocs gh-deploy --force

cm/CHANGES.md

@@ -1,3 +1,7 @@
+## V2.3.5
+   - added "cm init" to check system deps and pull mlcommons@cm4mlops by default
+   - fixed branch checkout in "cm pull repo"
+
 ## V2.3.4
    - minor documentation update
 

cm/README.md

@@ -195,7 +195,8 @@ cm pull repo --url=https://zenodo.org/records/12528908/files/cm4mlops-20240625.z
 cmr "install llvm prebuilt" --version=17.0.6
 cmr "app image corner-detection"
 
-cm run experiment --tags=tuning,experiment,batch_size -- echo --batch_size={{VAR1{range(1,8)}}}
+cm run experiment --tags=tuning,experiment,batch_size -- echo --batch_size={{ print_str("VAR1{range(1,8)}") }}
+
 cm replay experiment --tags=tuning,experiment,batch_size
 
 cmr "get conda"

cm/cmind/__init__.py

@@ -2,7 +2,7 @@
 #
 # Written by Grigori Fursin
 
-__version__ = "2.3.4"
+__version__ = "2.3.5"
 
 from cmind.core import access
 from cmind.core import error

cm/cmind/core.py

@@ -255,6 +255,9 @@ def access(self, i, out = None):
         elif action == '' and automation == '' and i.get('version',False):
             action = 'version'
             automation = 'core'
+        elif action == 'init' and automation == '':
+            automation = 'core'
+
 
         # Print basic help if action == ''
         extra_help = True if action == 'help' and automation == '' else False

cm/cmind/repo/automation/core/module.py

@@ -13,6 +13,166 @@
 #sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
 #from cm_60cb625a46b38610 import misc
 
+class CMInit():
+    ###############################################################
+    def run(self, quiet = False, repo_name = 'mlcommons@cm4mlops', repo_branch = ''):
+        import cmind
+
+        print ('Checking platform information ...')
+        self.get_sys_platform()
+
+        print ('')
+        print ('Checking system dependencies ...')
+        r = self.install_system_packages(quiet)
+        if r['return']>0 or r.get('warning','') !='' :
+            return r
+
+        print ('')
+        print ('Obtaining default automation repository ...')
+
+        print ('')
+        ii = {'action':'pull', 
+              'automation':'repo',
+              'artifact':repo_name,
+              'out':'con'}
+
+        if repo_branch !='':
+            ii['branch'] = repo_branch
+
+        return cmind.access(ii)
+
+    ###############################################################
+    def install_system_packages(self, quiet):
+
+        import sys
+        import importlib.util
+
+        # List of packages to install via system package manager
+        packages = []
+
+        git_status = self.command_exists('git')
+        if not git_status:
+            packages.append("git")
+
+        wget_status = self.command_exists('wget')
+        if not wget_status:
+            packages.append("wget")
+
+        curl_status = self.command_exists('curl')
+        if not curl_status:
+            packages.append("curl")
+
+        name='venv'
+
+        if name in sys.modules:
+            pass #nothing needed
+        else:
+            spec = importlib.util.find_spec(name)
+            if spec is not None:
+                module = importlib.util.module_from_spec(spec)
+                sys.modules[name] = module
+                spec.loader.exec_module(module)
+            else:
+                packages.append("python3-venv")
+
+        warning = ''
+
+        if packages:
+
+            install_cmd = ''
+
+            if self.system == 'Linux' or self.system == 'Darwin':
+                manager, details = self.get_package_manager_details()
+                if manager:
+                    if manager == "apt-get":
+                        install_cmd = '{}apt-get update && apt-get install -y {}'
+
+            if install_cmd == '':
+                warning = "You must install the following system packages manually: {}".format(', '.join(packages))
+            else:
+                print ('')
+                print ('The following system packages will be installed:')
+                print ('')
+                print (install_cmd.format('sudo', ' '.join(packages)))
+
+                sudo = 'sudo '
+                if not quiet:
+                    print ('')
+                    x = input ('Would you like to skip "sudo" from above command (y/N)? ')
+
+                    if x.lower() in ['y','yes']:
+                        sudo = ''
+
+                install_cmd = install_cmd.format(sudo, ' '.join(packages))
+
+                print ('')
+                print ('Running system command:')
+                print (install_cmd)
+
+                r = os.system(install_cmd)
+                if r>0:
+                    return {'return':1, 'error':f'Command {install_cmd} failed with return code {r}'}
+
+        rr = {'return':0}
+
+        if warning != '': rr['warning'] = warning
+
+        return rr
+
+    ###############################################################
+    def detect_package_manager(self):
+        package_managers = {
+            'apt-get': '/usr/bin/apt-get',
+            'yum': '/usr/bin/yum',
+            'dnf': '/usr/bin/dnf',
+            'pacman': '/usr/bin/pacman',
+            'zypper': '/usr/bin/zypper',
+            'brew': '/usr/local/bin/brew'
+        }
+
+        for name, path in package_managers.items():
+            if os.path.exists(path):
+                return name
+
+        return None
+
+    ###############################################################
+    def get_package_manager_details(self):
+        import subprocess
+
+        manager = self.detect_package_manager()
+        if manager:
+            try:
+                version_output = subprocess.check_output([manager, '--version'], stderr=subprocess.STDOUT).decode('utf-8')
+                return manager, version_output.split('\n')[0]
+            except subprocess.CalledProcessError:
+                return manager, 'Version information not available'
+        else:
+            return None, 'No supported package manager found'
+
+    ###############################################################
+    # Checks if command exists(for installing required packages).
+    # If the command exists, which returns 0, making the function return True.
+    # If the command does not exist, which returns a non-zero value, making the function return False.
+    # NOTE: The standard output and standard error streams are redirected to PIPES so that it could be captured in future if needed.    
+    def command_exists(self, command):
+        import subprocess
+
+        if self.system == "Linux" or self.system == 'Darwin':
+            return subprocess.call(['which', command], stdout=subprocess.PIPE, stderr=subprocess.PIPE) == 0
+
+        elif self.system == "Windows":
+            return subprocess.call([command, '--version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) == 0
+
+
+    ###############################################################
+    def get_sys_platform(self):
+        import platform
+
+        self.system =  platform.system() 
+
+
+
 class CAutomation(Automation):
     """
     CM "core" automation actions
@@ -42,3 +202,41 @@ def uid(self, i):
 
         return utils.call_internal_module(self, __file__, 'module_misc', 'uid', i)
 
+    ############################################################
+    def init(self, i):
+        """
+        Init CM
+
+        Args:
+          (CM input dict):
+
+          (quiet) (bool): if True, skip asking questions about sudo, etc
+          (repo) (str): automation repository to pull ('mlcommons@cm4mlops' by default) 
+          (branch) (str): branch to use ('' by default)
+
+
+        Returns:
+          (CM return dict):
+
+          * return (int): return code == 0 if no error and >0 if error
+          * (error) (str): error string if return>0
+        """
+
+        cm_init = CMInit()
+
+        quiet = i.get('quiet', False)
+
+        repo_name = i.get('repo', '')
+        if repo_name == '': repo_name = 'mlcommons@cm4mlops'
+
+        repo_branch = i.get('branch', '')
+
+        r = cm_init.run(quiet = quiet, repo_name = repo_name, repo_branch = repo_branch)
+        if r['return']>0: return r
+
+        warning = r.get('warning', '')
+        if warning != '':
+            print ('')
+            print (warning)
+
+        return {'return':0}

dev/meetings/20240731.md

@@ -18,7 +18,7 @@ using these examples:
 * https://cython.org (see Financial contributions section)
 
 * Author/creator
-* Core developers
+* Core developers: 
 * Contributors (from the community, MLCommons and the Automation and Reproducibility TaskForce):
   See https://github.com/mlcommons/ck/blob/master/CONTRIBUTING.md .
 * Sponsorship & financial Contributions

dev/meetings/20240808.md

@@ -0,0 +1,22 @@
+# Topic
+
+Urgent tasks
+
+# People
+
+* Grigori Fursin
+* Arjun Suresh
+
+# TBD
+
+## Finish proper attribution
+
+For CM, CM4MLOps and CM4ABTF
+
+## Add cm init and release CM
+
+## Check default MLPerf repos (MLCommons)
+
+## How is responding at Discrod
+
+Anandhu ?

docs/getting-started.md

@@ -2,32 +2,6 @@
 
 # Collective Mind Getting Started Guide and FAQ
 
-<details>
-<summary>Click here to see the table of contents.</summary>
-
-* [Collective Mind Getting Started Guide and FAQ](#collective-mind-getting-started-guide-and-faq)
-  * [Why CM?](#why-cm?)
-  * [CM automation recipe for image classification](#cm-automation-recipe-for-image-classification)
-  * [How CM scripts works?](#how-cm-scripts-works?)
-  * [How CM runs automation recipes?](#how-cm-runs-automation-recipes?)
-  * [How CM unifies inputs, outputs and environment variables?](#how-cm-unifies-inputs-outputs-and-environment-variables?)
-  * [How CM chains automation recipes into portable workflows?](#how-cm-chains-automation-recipes-into-portable-workflows?)
-  * [How to add new CM scripts?](#how-to-add-new-cm-scripts?)
-  * [How to customize CM scripts using variations?](#how-to-customize-cm-scripts-using-variations?)
-  * [How to cache and reuse CM scripts' output?](#how-to-cache-and-reuse-cm-scripts'-output?)
-  * [How to use CM with Python virtual environments?](#how-to-use-cm-with-python-virtual-environments?)
-  * [How to debug CM scripts?](#how-to-debug-cm-scripts?)
-  * [How to extend/improve CM scripts?](#how-to-extend/improve-cm-scripts?)
-  * [How to use CM with containers?](#how-to-use-cm-with-containers?)
-  * [How to use CM GUI to run automation recipes?](#how-to-use-cm-gui-to-run-automation-recipes?)
-  * [How to run MLPerf benchmarks via CM?](#how-to-run-mlperf-benchmarks-via-cm?)
-  * [How to use CM to reproduce research papers?](#how-to-use-cm-to-reproduce-research-papers?)
-  * [How to use CM as a common interface to other projects?](#how-to-use-cm-as-a-common-interface-to-other-projects?)
-  * [Where to read about the CM vision and history?](#where-to-read-about-the-cm-vision-and-history?)
-  * [How to get in touch with the CM community?](#how-to-get-in-touch-with-the-cm-community?)
-
-</details>
-
 
 ## Why CM?
 

docs/index.md

@@ -0,0 +1 @@
+../cm/README.md