From 01680638d9acab85c96eb879e63b76b6eb658b63 Mon Sep 17 00:00:00 2001 From: baptiste Date: Wed, 21 Aug 2024 14:19:58 +0000 Subject: [PATCH] add intel pytorch ort and openvino to leaderboard --- .github/workflows/update_llm_perf_cuda_pytorch.yaml | 2 +- llm_perf/hardware/intel/update_llm_perf_intel_openvino.py | 4 +--- llm_perf/hardware/intel/update_llm_perf_intel_ort.py | 4 +--- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/.github/workflows/update_llm_perf_cuda_pytorch.yaml b/.github/workflows/update_llm_perf_cuda_pytorch.yaml index 0ab646ab..567128e5 100644 --- a/.github/workflows/update_llm_perf_cuda_pytorch.yaml +++ b/.github/workflows/update_llm_perf_cuda_pytorch.yaml @@ -53,4 +53,4 @@ jobs: pip install packaging && pip install flash-attn einops scipy auto-gptq optimum bitsandbytes autoawq codecarbon pip install -U transformers huggingface_hub[hf_transfer] pip install -e . - python llm_perf/update_llm_perf_cuda_pytorch.py + python llm_perf/hardware/cuda/update_llm_perf_cuda_pytorch.py diff --git a/llm_perf/hardware/intel/update_llm_perf_intel_openvino.py b/llm_perf/hardware/intel/update_llm_perf_intel_openvino.py index 1aa6052a..86959828 100644 --- a/llm_perf/hardware/intel/update_llm_perf_intel_openvino.py +++ b/llm_perf/hardware/intel/update_llm_perf_intel_openvino.py @@ -12,7 +12,7 @@ is_benchmark_conducted, is_benchmark_supported, ) -from optimum_benchmark import Benchmark, BenchmarkConfig, BenchmarkReport, InferenceConfig, ProcessConfig, OVConfig +from optimum_benchmark import Benchmark, BenchmarkConfig, BenchmarkReport, InferenceConfig, OVConfig, ProcessConfig from optimum_benchmark.logging_utils import setup_logging SUBSET = os.getenv("SUBSET", None) @@ -101,8 +101,6 @@ def benchmark_intel_openvino(model, attn_implementation, weights_config): benchmark_name = f"{weights_config}-{attn_implementation}" subfolder = f"{benchmark_name}/{model.replace('/', '--')}" - torch_dtype = WEIGHTS_CONFIGS[weights_config]["torch_dtype"] - quant_scheme = WEIGHTS_CONFIGS[weights_config]["quant_scheme"] quant_config = WEIGHTS_CONFIGS[weights_config]["quant_config"] if not is_benchmark_supported(weights_config, attn_implementation, HARDWARE): diff --git a/llm_perf/hardware/intel/update_llm_perf_intel_ort.py b/llm_perf/hardware/intel/update_llm_perf_intel_ort.py index e7bb254c..5b8f5c3e 100644 --- a/llm_perf/hardware/intel/update_llm_perf_intel_ort.py +++ b/llm_perf/hardware/intel/update_llm_perf_intel_ort.py @@ -1,5 +1,4 @@ import os -import traceback from itertools import product from logging import getLogger @@ -12,7 +11,7 @@ is_benchmark_conducted, is_benchmark_supported, ) -from optimum_benchmark import Benchmark, BenchmarkConfig, BenchmarkReport, InferenceConfig, ProcessConfig, ORTConfig +from optimum_benchmark import Benchmark, BenchmarkConfig, InferenceConfig, ORTConfig, ProcessConfig from optimum_benchmark.logging_utils import setup_logging SUBSET = os.getenv("SUBSET", None) @@ -102,7 +101,6 @@ def benchmark_intel_ort(model, attn_implementation, weights_config): subfolder = f"{benchmark_name}/{model.replace('/', '--')}" torch_dtype = WEIGHTS_CONFIGS[weights_config]["torch_dtype"] - quant_scheme = WEIGHTS_CONFIGS[weights_config]["quant_scheme"] quant_config = WEIGHTS_CONFIGS[weights_config]["quant_config"] if not is_benchmark_supported(weights_config, attn_implementation, HARDWARE):