Skip to content

Commit

Permalink
Add t4 for llm perf leaderboard (#238)
Browse files Browse the repository at this point in the history
  • Loading branch information
baptistecolle authored Aug 19, 2024
1 parent bca546a commit 0b69851
Show file tree
Hide file tree
Showing 5 changed files with 12 additions and 4 deletions.
6 changes: 5 additions & 1 deletion .github/workflows/update_llm_perf_cuda_pytorch.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,11 @@ jobs:
fail-fast: false
matrix:
subset: [unquantized, bnb, awq, gptq]
machine: [{ name: 1xA10, runs-on: [single-gpu, nvidia-gpu, a10, ci] }]

machine: [
{name: 1xA10, runs-on: {group: 'aws-g5-4xlarge-plus'}},
{name: 1xT4, runs-on: {group: 'aws-g4dn-2xlarge'}}
]

runs-on: ${{ matrix.machine.runs-on }}

Expand Down
2 changes: 1 addition & 1 deletion llm_perf/update_llm_perf_cuda_pytorch.py
Original file line number Diff line number Diff line change
Expand Up @@ -134,7 +134,7 @@ def benchmark_cuda_pytorch(model, attn_implementation, weights_config):
quantization_scheme=quant_scheme,
quantization_config=quant_config,
attn_implementation=attn_implementation,
hub_kwargs={"trust_remote_code": True},
model_kwargs={"trust_remote_code": True},
)

benchmark_config = BenchmarkConfig(
Expand Down
2 changes: 1 addition & 1 deletion llm_perf/update_llm_perf_leaderboard.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ def gather_benchmarks(subset: str, machine: str):

def update_perf_dfs():
for subset in ["unquantized", "bnb", "awq", "gptq"]:
for machine in ["1xA10", "1xA100"]:
for machine in ["1xA10", "1xA100", "1xT4"]:
try:
gather_benchmarks(subset, machine)
except Exception:
Expand Down
1 change: 1 addition & 0 deletions optimum_benchmark/backends/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@ def __post_init__(self):
self.library,
revision=self.model_kwargs.get("revision", None),
token=self.model_kwargs.get("token", None),
trust_remote_code=self.model_kwargs.get("trust_remote_code", False),
)

if self.device is None:
Expand Down
5 changes: 4 additions & 1 deletion optimum_benchmark/task_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -190,6 +190,7 @@ def infer_model_type_from_model_name_or_path(
library_name: Optional[str] = None,
revision: Optional[str] = None,
token: Optional[str] = None,
trust_remote_code: bool = False,
) -> str:
if library_name is None:
library_name = infer_library_from_model_name_or_path(model_name_or_path, revision=revision, token=token)
Expand All @@ -216,7 +217,9 @@ def infer_model_type_from_model_name_or_path(
break

else:
transformers_config = get_transformers_pretrained_config(model_name_or_path, revision=revision, token=token)
transformers_config = get_transformers_pretrained_config(
model_name_or_path, revision=revision, token=token, trust_remote_code=trust_remote_code
)
inferred_model_type = transformers_config.model_type

if inferred_model_type is None:
Expand Down

0 comments on commit 0b69851

Please sign in to comment.