diff --git a/script/app-mlperf-inference/customize.py b/script/app-mlperf-inference/customize.py index 6ab5bb51da..0e18dea61e 100644 --- a/script/app-mlperf-inference/customize.py +++ b/script/app-mlperf-inference/customize.py @@ -33,6 +33,9 @@ def preprocess(i): run_state = i['run_script_input']['run_state'] state['mlperf-inference-implementation']['script_id'] = run_state['script_id']+":"+",".join(run_state['script_variation_tags']) + if env.get('CM_VLLM_SERVER_MODEL_NAME', '') != '' and env.get('CM_ML_MODEL_FULL_NAME', '') == '': + env['CM_ML_MODEL_FULL_NAME'] = env['CM_VLLM_SERVER_MODEL_NAME'].replace("/", "_") + return {'return':0} def postprocess(i): @@ -288,9 +291,9 @@ def postprocess(i): cmd = "" xcmd = "" - readme_init = "This experiment is generated using the [MLCommons Collective Mind automation framework (CM)](https://github.com/mlcommons/ck).\n\n" + readme_init = "This experiment is generated using the [MLCommons Collective Mind automation framework (CM)](https://github.com/mlcommons/cm4mlops).\n\n" - readme_init+= "*Check [CM MLPerf docs](https://mlcommons.github.io/inference) for more details.*\n\n" + readme_init+= "*Check [CM MLPerf docs](https://docs.mlcommons.org/inference) for more details.*\n\n" readme_body = "## Host platform\n\n* OS version: {}\n* CPU version: {}\n* Python version: {}\n* MLCommons CM version: {}\n\n".format(platform.platform(), platform.processor(), sys.version, cm.__version__) @@ -298,10 +301,10 @@ def postprocess(i): x = repo_name if repo_hash!='': x+=' --checkout='+str(repo_hash) - readme_body += "## CM Run Command\n\nSee [CM installation guide](https://github.com/mlcommons/ck/blob/master/docs/installation.md).\n\n"+ \ + readme_body += "## CM Run Command\n\nSee [CM installation guide](https://docs.mlcommons.org/inference/install/).\n\n"+ \ "```bash\npip install -U cmind\n\ncm rm cache -f\n\ncm pull repo {}\n\n{}\n```".format(x, xcmd) - readme_body += "\n*Note that if you want to use the [latest automation recipes](https://access.cknowledge.org/playground/?action=scripts) for MLPerf (CM scripts),\n"+ \ + readme_body += "\n*Note that if you want to use the [latest automation recipes](https://docs.mlcommons.org/inference) for MLPerf (CM scripts),\n"+ \ " you should simply reload {} without checkout and clean CM cache as follows:*\n\n".format(repo_name) + \ "```bash\ncm rm repo {}\ncm pull repo {}\ncm rm cache -f\n\n```".format(repo_name, repo_name) @@ -415,9 +418,6 @@ def postprocess(i): is_valid = checker.check_compliance_perf_dir(COMPLIANCE_DIR) state['cm-mlperf-inference-results'][state['CM_SUT_CONFIG_NAME']][model][scenario][test] = "passed" if is_valid else "failed" - else: - print(test) - if state.get('mlperf-inference-implementation') and state['mlperf-inference-implementation'].get('version_info'): with open(os.path.join(output_dir, "cm-version-info.json"), "w") as f: diff --git a/script/get-mlperf-inference-utils/mlperf_utils.py b/script/get-mlperf-inference-utils/mlperf_utils.py index 8682c10660..3a38302374 100644 --- a/script/get-mlperf-inference-utils/mlperf_utils.py +++ b/script/get-mlperf-inference-utils/mlperf_utils.py @@ -70,16 +70,22 @@ def get_accuracy_metric(config, model, path): acc_upper_limit = config.get_accuracy_upper_limit(model) patterns = [] acc_targets = [] - acc_limits = [] - up_patterns = [] + acc_limits = [None] * (len(target)//2) + up_patterns = [None] * (len(target)//2) acc_types = [] if acc_upper_limit is not None: acc_limit_check = True - for i in range(0, len(acc_upper_limit), 2): - acc_type, acc_target = acc_upper_limit[i:i+2] - acc_limits.append(acc_target) - up_patterns.append(checker.ACC_PATTERN[acc_type]) + + for ii in range(0, len(target), 2): + acc_type1,tmp = target[ii:ii+2] + for i in range(0, len(acc_upper_limit), 2): + acc_type, acc_target = acc_upper_limit[i:i+2] + if acc_type != acc_type1: + continue + acc_limits[ii//2] = acc_target + up_patterns[ii//2] = checker.ACC_PATTERN[acc_type] + for i in range(0, len(target), 2): acc_type, acc_target = target[i:i+2] @@ -109,6 +115,8 @@ def get_accuracy_metric(config, model, path): acc = None if acc_upper_limit is not None: for i, (pattern, acc_limit) in enumerate(zip(up_patterns, acc_limits)): + if not pattern: + continue m = re.match(pattern, line) if m: acc = m.group(1) @@ -197,7 +205,7 @@ def get_result_string(version, model, scenario, result_path, has_power, sub_res, for i, acc in enumerate(acc_results): accuracy_results.append(str(round(float(acc_results[acc]), 5))) accuracy_result_string += f"`{acc}`: `{round(float(acc_results[acc]), 5)}`" - if not acc_limits: + if not acc_limits or not acc_limits[i]: accuracy_result_string += f", Required accuracy for closed division `>= {round(acc_targets[i], 5)}`" else: accuracy_result_string += f", Required accuracy for closed division `>= {round(acc_targets[i], 5)}` and `<= {round(acc_limits[i], 5)}`" diff --git a/script/run-mlperf-inference-app/customize.py b/script/run-mlperf-inference-app/customize.py index 67a60daabf..d8a1fe8db2 100644 --- a/script/run-mlperf-inference-app/customize.py +++ b/script/run-mlperf-inference-app/customize.py @@ -101,10 +101,13 @@ def preprocess(i): test_list = ["TEST01", "TEST05"] if env['CM_MODEL'] in ["resnet50"]: test_list.append("TEST04") - if "gpt" in env['CM_MODEL'] or "sdxl" in env['CM_MODEL'] or "llama2-70b" in env['CM_MODEL'] or "mixtral-8x7b" in env['CM_MODEL']: + if "gpt" in env['CM_MODEL'] or "llama2-70b" in env['CM_MODEL'] or "mixtral-8x7b" in env['CM_MODEL']: test_list.remove("TEST01") test_list.remove("TEST05") + if "llama2-70b" in env['CM_MODEL'] or "mixtral-8x7b" in env['CM_MODEL']: + test_list.append("TEST06") + variation_implementation= "_" + env.get("CM_MLPERF_IMPLEMENTATION", "reference") variation_model= ",_" + env["CM_MLPERF_MODEL"] variation_backend= ",_" + env["CM_MLPERF_BACKEND"] if env.get("CM_MLPERF_BACKEND","") != "" else ""