Skip to content

Commit

Permalink
fix metrics
Browse files Browse the repository at this point in the history
  • Loading branch information
@Samoed
Samoed committed Nov 15, 2024
1 parent 29546d7 commit 186d69a
Show file tree
Hide file tree
Showing 14,475 changed files with 393,562 additions and 393,559 deletions.
The diff you're trying to view is too large. We only load the first 3000 changed files.
7 changes: 5 additions & 2 deletions load_external.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -169,7 +169,10 @@ def parse_readme(model_info: ModelInfo) -> dict[str, dict[str, Any]] | None:
"languages": mteb_task_eval_languages if isinstance(mteb_task_eval_languages, list) else mteb_task_eval_languages.get(current_config, ["None"]),
}
for metric in result["metrics"]:
cur_split_metrics[metric["type"]] = metric["value"]
if isinstance(metric["value"], (float, int)):
cur_split_metrics[metric["type"]] = metric["value"] / 100
else:
cur_split_metrics[metric["type"]] = metric["value"]

main_score_str = "main_score"
if main_score_str not in cur_split_metrics:
Expand Down Expand Up @@ -198,7 +201,7 @@ def get_mteb_data() -> None:
model_path = get_model_dir(model_info.id)
if (model_path / "model_meta.json").exists() and len(list(model_path.glob("*.json"))) > 1:
logger.info(f"Model meta already exists for {model_info.id}")
continue
# continue
if model_info.id.lower().endswith("gguf"):
logger.info(f"Skipping {model_info.id} GGUF model")
continue
Expand Down
14 changes: 7 additions & 7 deletions results/AbderrahmanSkiredj1__Arabic_text_embedding_for_sts/external/STS17.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,13 +10,13 @@
"languages": [
"ara-Arab"
],
"cosine_pearson": 84.5661961200634,
"cosine_spearman": 85.04839486363485,
"euclidean_pearson": 83.06651812356505,
"euclidean_spearman": 84.64285835335832,
"main_score": 85.04839486363485,
"manhattan_pearson": 83.12273085900199,
"manhattan_spearman": 84.51258675131285
"cosine_pearson": 0.845661961200634,
"cosine_spearman": 0.8504839486363486,
"euclidean_pearson": 0.8306651812356505,
"euclidean_spearman": 0.8464285835335832,
"main_score": 0.8504839486363486,
"manhattan_pearson": 0.8312273085900199,
"manhattan_spearman": 0.8451258675131286
}
]
}
Expand Down
14 changes: 7 additions & 7 deletions ...rahmanSkiredj1__arabic_text_embedding_sts_arabertv02_arabicnlitriplet/external/STS17.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,13 +10,13 @@
"languages": [
"ara-Arab"
],
"cosine_pearson": 84.4552565985224,
"cosine_spearman": 84.95895784502326,
"euclidean_pearson": 82.78929391924628,
"euclidean_spearman": 84.30172209203968,
"main_score": 84.95895784502326,
"manhattan_pearson": 82.8787314198733,
"manhattan_spearman": 84.31971710672602
"cosine_pearson": 0.844552565985224,
"cosine_spearman": 0.8495895784502326,
"euclidean_pearson": 0.8278929391924628,
"euclidean_spearman": 0.8430172209203968,
"main_score": 0.8495895784502326,
"manhattan_pearson": 0.8287873141987331,
"manhattan_spearman": 0.8431971710672602
}
]
}
Expand Down
8 changes: 4 additions & 4 deletions results/AdrienB134__llm2vec-occiglot-mntp/external/AlloProfClusteringP2P.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
"languages": [
"fra-Latn"
],
"v_measure": 63.99706100543374,
"v_measure": 0.6399706100543374,
"v_measures": [
0.6309250305032879,
0.6712141669904825,
Expand Down Expand Up @@ -1013,14 +1013,14 @@
0.6312400213373579,
0.6765111319025582
],
"main_score": 63.99706100543374
"main_score": 0.6399706100543374
},
{
"hf_subset": "default",
"languages": [
"fra-Latn"
],
"v_measure": 29.932569372425572,
"v_measure": 0.2993256937242557,
"v_measures": [
0.27074628475701645,
0.3138981270131547,
Expand Down Expand Up @@ -2023,7 +2023,7 @@
0.2655556969048289,
0.2624750092062693
],
"main_score": 29.932569372425572
"main_score": 0.2993256937242557
}
]
}
Expand Down
6 changes: 3 additions & 3 deletions results/AdrienB134__llm2vec-occiglot-mntp/external/AlloprofReranking.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,9 @@
"languages": [
"fra-Latn"
],
"map": 37.09270648443667,
"mrr": 37.65235627929929,
"main_score": 37.09270648443667
"map": 0.37092706484436666,
"mrr": 0.3765235627929929,
"main_score": 0.37092706484436666
}
]
}
Expand Down
6 changes: 3 additions & 3 deletions results/AdrienB134__llm2vec-occiglot-mntp/external/AmazonReviewsClassification.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,8 @@
"languages": [
"fra-Latn"
],
"accuracy": 35.071999999999996,
"f1": 34.756733638269665,
"accuracy": 0.35072,
"f1": 0.34756733638269666,
"scores_per_experiment": [
{
"accuracy": 0.3344,
Expand Down Expand Up @@ -4014,7 +4014,7 @@
"f1": 0.3516757627896389
}
],
"main_score": 35.071999999999996
"main_score": 0.35072
}
]
}
Expand Down
72 changes: 36 additions & 36 deletions results/AdrienB134__llm2vec-occiglot-mntp/external/BSARDRetrieval.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,42 +10,42 @@
"languages": [
"fra-Latn"
],
"map_at_1": 0.901,
"map_at_10": 1.6150000000000002,
"map_at_100": 2.046,
"map_at_1000": 2.147,
"map_at_20": 1.8030000000000002,
"map_at_3": 1.126,
"map_at_5": 1.486,
"mrr_at_1": 0.901,
"mrr_at_10": 1.6150000000000002,
"mrr_at_100": 2.046,
"mrr_at_1000": 2.147,
"mrr_at_20": 1.8030000000000002,
"mrr_at_3": 1.126,
"mrr_at_5": 1.486,
"ndcg_at_1": 0.901,
"ndcg_at_10": 2.182,
"ndcg_at_100": 4.926,
"ndcg_at_1000": 7.825,
"ndcg_at_20": 2.8649999999999998,
"ndcg_at_3": 1.185,
"ndcg_at_5": 1.882,
"precision_at_1": 0.901,
"precision_at_10": 0.40499999999999997,
"precision_at_100": 0.185,
"precision_at_1000": 0.042,
"precision_at_20": 0.338,
"precision_at_3": 0.44999999999999996,
"precision_at_5": 0.631,
"recall_at_1": 0.901,
"recall_at_10": 4.054,
"recall_at_100": 18.468,
"recall_at_1000": 41.892,
"recall_at_20": 6.757000000000001,
"recall_at_3": 1.351,
"recall_at_5": 3.1530000000000005,
"main_score": 18.468
"map_at_1": 0.00901,
"map_at_10": 0.01615,
"map_at_100": 0.02046,
"map_at_1000": 0.02147,
"map_at_20": 0.01803,
"map_at_3": 0.01126,
"map_at_5": 0.01486,
"mrr_at_1": 0.00901,
"mrr_at_10": 0.01615,
"mrr_at_100": 0.02046,
"mrr_at_1000": 0.02147,
"mrr_at_20": 0.01803,
"mrr_at_3": 0.01126,
"mrr_at_5": 0.01486,
"ndcg_at_1": 0.00901,
"ndcg_at_10": 0.02182,
"ndcg_at_100": 0.04926,
"ndcg_at_1000": 0.07825,
"ndcg_at_20": 0.02865,
"ndcg_at_3": 0.011850000000000001,
"ndcg_at_5": 0.01882,
"precision_at_1": 0.00901,
"precision_at_10": 0.00405,
"precision_at_100": 0.00185,
"precision_at_1000": 0.00042,
"precision_at_20": 0.00338,
"precision_at_3": 0.0045,
"precision_at_5": 0.00631,
"recall_at_1": 0.00901,
"recall_at_10": 0.04054,
"recall_at_100": 0.18468,
"recall_at_1000": 0.41892,
"recall_at_20": 0.06757,
"recall_at_3": 0.01351,
"recall_at_5": 0.03153,
"main_score": 0.18468
}
]
}
Expand Down
4 changes: 2 additions & 2 deletions results/AdrienB134__llm2vec-occiglot-mntp/external/HALClusteringS2S.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
"languages": [
"fra-Latn"
],
"v_measure": 20.821362086725557,
"v_measure": 0.20821362086725556,
"v_measures": [
0.22505579417192464,
0.18718591610641097,
Expand Down Expand Up @@ -1013,7 +1013,7 @@
0.20352917236688817,
0.2491962641585171
],
"main_score": 20.821362086725557
"main_score": 0.20821362086725556
}
]
}
Expand Down
8 changes: 4 additions & 4 deletions results/AdrienB134__llm2vec-occiglot-mntp/external/MLSUMClusteringP2P.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
"languages": [
"fra-Latn"
],
"v_measure": 45.2570658673633,
"v_measure": 0.452570658673633,
"v_measures": [
0.4329487970215975,
0.45820580504076386,
Expand Down Expand Up @@ -1013,14 +1013,14 @@
0.44613674850966567,
0.43952701472303013
],
"main_score": 45.2570658673633
"main_score": 0.452570658673633
},
{
"hf_subset": "fr",
"languages": [
"fra-Latn"
],
"v_measure": 44.94685216731792,
"v_measure": 0.4494685216731792,
"v_measures": [
0.4451988218629661,
0.44546586627639506,
Expand Down Expand Up @@ -2023,7 +2023,7 @@
0.4562406688345804,
0.43126055742654296
],
"main_score": 44.94685216731792
"main_score": 0.4494685216731792
}
]
}
Expand Down
6 changes: 3 additions & 3 deletions results/AdrienB134__llm2vec-occiglot-mntp/external/MTOPDomainClassification.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,8 @@
"languages": [
"fra-Latn"
],
"accuracy": 75.69996868149076,
"f1": 75.40911375049545,
"accuracy": 0.7569996868149076,
"f1": 0.7540911375049545,
"scores_per_experiment": [
{
"accuracy": 0.7788913247729408,
Expand Down Expand Up @@ -4014,7 +4014,7 @@
"f1": 0.7570027586618345
}
],
"main_score": 75.69996868149076
"main_score": 0.7569996868149076
}
]
}
Expand Down
6 changes: 3 additions & 3 deletions results/AdrienB134__llm2vec-occiglot-mntp/external/MTOPIntentClassification.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,8 @@
"languages": [
"fra-Latn"
],
"accuracy": 63.75822110867524,
"f1": 42.22367596410695,
"accuracy": 0.6375822110867524,
"f1": 0.42223675964106955,
"scores_per_experiment": [
{
"accuracy": 0.6623864704040088,
Expand Down Expand Up @@ -4014,7 +4014,7 @@
"f1": 0.41098404682577316
}
],
"main_score": 63.75822110867524
"main_score": 0.6375822110867524
}
]
}
Expand Down
12 changes: 6 additions & 6 deletions results/AdrienB134__llm2vec-occiglot-mntp/external/MasakhaNEWSClassification.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,8 @@
"languages": [
"eng-Latn"
],
"accuracy": 77.91139240506328,
"f1": 77.05013060581847,
"accuracy": 0.7791139240506328,
"f1": 0.7705013060581847,
"scores_per_experiment": [
{
"accuracy": 0.8037974683544303,
Expand Down Expand Up @@ -4014,15 +4014,15 @@
"f1": 0.8019194870291454
}
],
"main_score": 77.91139240506328
"main_score": 0.7791139240506328
},
{
"hf_subset": "fra",
"languages": [
"fra-Latn"
],
"accuracy": 75.9952606635071,
"f1": 72.14436891647776,
"accuracy": 0.7599526066350709,
"f1": 0.7214436891647776,
"scores_per_experiment": [
{
"accuracy": 0.7890995260663507,
Expand Down Expand Up @@ -8025,7 +8025,7 @@
"f1": 0.6562022376789257
}
],
"main_score": 75.9952606635071
"main_score": 0.7599526066350709
}
]
}
Expand Down
6 changes: 3 additions & 3 deletions results/AdrienB134__llm2vec-occiglot-mntp/external/MassiveIntentClassification.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,8 @@
"languages": [
"fra-Latn"
],
"accuracy": 56.028917283120386,
"f1": 52.65339345173721,
"accuracy": 0.5602891728312038,
"f1": 0.5265339345173721,
"scores_per_experiment": [
{
"accuracy": 0.5531271015467384,
Expand Down Expand Up @@ -4014,7 +4014,7 @@
"f1": 0.5358281437791133
}
],
"main_score": 56.028917283120386
"main_score": 0.5602891728312038
}
]
}
Expand Down
6 changes: 3 additions & 3 deletions results/AdrienB134__llm2vec-occiglot-mntp/external/MassiveScenarioClassification.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,8 @@
"languages": [
"fra-Latn"
],
"accuracy": 59.297242770679226,
"f1": 57.613317912056885,
"accuracy": 0.5929724277067923,
"f1": 0.5761331791205688,
"scores_per_experiment": [
{
"accuracy": 0.5938130464021519,
Expand Down Expand Up @@ -4014,7 +4014,7 @@
"f1": 0.583012423695179
}
],
"main_score": 59.297242770679226
"main_score": 0.5929724277067923
}
]
}
Expand Down
Loading

0 comments on commit 186d69a

Please sign in to comment.