Add Mean Average Precision (MAP) metric (#338)

cornac/metrics/__init__.py

@@ -13,6 +13,12 @@
 # limitations under the License.
 # ============================================================================
 
+
+from .rating import RatingMetric
+from .rating import MAE
+from .rating import RMSE
+from .rating import MSE
+
 from .ranking import RankingMetric
 from .ranking import NDCG
 from .ranking import NCRR
@@ -21,8 +27,4 @@
 from .ranking import Recall
 from .ranking import FMeasure
 from .ranking import AUC
-
-from .rating import RatingMetric
-from .rating import MAE
-from .rating import RMSE
-from .rating import MSE
+from .ranking import MAP

cornac/metrics/ranking.py

@@ -14,6 +14,7 @@
 # ============================================================================
 
 import numpy as np
+from scipy.stats import rankdata
 
 
 class RankingMetric:
@@ -34,9 +35,9 @@ class RankingMetric:
     """
 
     def __init__(self, name=None, k=-1, higher_better=True):
-        assert hasattr(k, '__len__') or k == -1 or k > 0
+        assert hasattr(k, "__len__") or k == -1 or k > 0
 
-        self.type = 'ranking'
+        self.type = "ranking"
         self.name = name
         self.k = k
         self.higher_better = higher_better
@@ -61,7 +62,7 @@ class NDCG(RankingMetric):
     """
 
     def __init__(self, k=-1):
-        RankingMetric.__init__(self, name='NDCG@{}'.format(k), k=k)
+        RankingMetric.__init__(self, name="NDCG@{}".format(k), k=k)
 
     @staticmethod
     def dcg_score(gt_pos, pd_rank, k=-1):
@@ -134,7 +135,7 @@ class NCRR(RankingMetric):
     """
 
     def __init__(self, k=-1):
-        RankingMetric.__init__(self, name='NCRR@{}'.format(k), k=k)
+        RankingMetric.__init__(self, name="NCRR@{}".format(k), k=k)
 
     def compute(self, gt_pos, pd_rank, **kwargs):
         """Compute Normalized Cumulative Reciprocal Rank score.
@@ -156,7 +157,7 @@ def compute(self, gt_pos, pd_rank, **kwargs):
 
         """
         if self.k > 0:
-            truncated_pd_rank = pd_rank[:self.k]
+            truncated_pd_rank = pd_rank[: self.k]
         else:
             truncated_pd_rank = pd_rank
 
@@ -167,13 +168,13 @@ def compute(self, gt_pos, pd_rank, **kwargs):
         if len(rec_rank) == 0:
             return 0.0
         rec_rank = rec_rank + 1  # +1 because indices starts from 0 in python
-        crr = np.sum(1. / rec_rank)
+        crr = np.sum(1.0 / rec_rank)
 
         # Compute Ideal CRR
-        max_nb_pos = min(len(gt_pos_items[0]),len(truncated_pd_rank))
+        max_nb_pos = min(len(gt_pos_items[0]), len(truncated_pd_rank))
         ideal_rank = np.arange(max_nb_pos)
         ideal_rank = ideal_rank + 1  # +1 because indices starts from 0 in python
-        icrr = np.sum(1. / ideal_rank)
+        icrr = np.sum(1.0 / ideal_rank)
 
         # Compute nDCG
         ncrr_i = crr / icrr
@@ -190,7 +191,7 @@ class MRR(RankingMetric):
     """
 
     def __init__(self):
-        RankingMetric.__init__(self, name='MRR')
+        RankingMetric.__init__(self, name="MRR")
 
     def compute(self, gt_pos, pd_rank, **kwargs):
         """Compute Mean Reciprocal Rank score.
@@ -215,9 +216,13 @@ def compute(self, gt_pos, pd_rank, **kwargs):
         matched_items = np.nonzero(np.in1d(pd_rank, gt_pos_items))[0]
 
         if len(matched_items) == 0:
-            raise ValueError('No matched between ground-truth items and recommendations')
+            raise ValueError(
+                "No matched between ground-truth items and recommendations"
+            )
 
-        mrr = np.divide(1, (matched_items[0] + 1))  # +1 because indices start from 0 in python
+        mrr = np.divide(
+            1, (matched_items[0] + 1)
+        )  # +1 because indices start from 0 in python
         return mrr
 
 
@@ -261,7 +266,7 @@ def compute(self, gt_pos, pd_rank, **kwargs):
 
         """
         if self.k > 0:
-            truncated_pd_rank = pd_rank[:self.k]
+            truncated_pd_rank = pd_rank[: self.k]
         else:
             truncated_pd_rank = pd_rank
 
@@ -404,7 +409,7 @@ class AUC(RankingMetric):
     """
 
     def __init__(self):
-        RankingMetric.__init__(self, name='AUC')
+        RankingMetric.__init__(self, name="AUC")
 
     def compute(self, pd_scores, gt_pos, gt_neg=None, **kwargs):
         """Compute Area Under the ROC Curve (AUC).
@@ -438,3 +443,42 @@ def compute(self, pd_scores, gt_pos, gt_neg=None, **kwargs):
         uj_scores = np.tile(neg_scores, len(pos_scores))
 
         return (ui_scores > uj_scores).sum() / len(uj_scores)
+
+
+class MAP(RankingMetric):
+    """Mean Average Precision (MAP).
+
+    References
+    ----------
+    https://en.wikipedia.org/wiki/Evaluation_measures_(information_retrieval)#Mean_average_precision
+
+    """
+
+    def __init__(self):
+        RankingMetric.__init__(self, name="MAP")
+
+    def compute(self, pd_scores, gt_pos, **kwargs):
+        """Compute Average Precision.
+
+        Parameters
+        ----------
+        pd_scores: Numpy array
+            Prediction scores for items.
+
+        gt_pos: Numpy array
+            Binary vector of positive items.
+
+        **kwargs: For compatibility
+
+        Returns
+        -------
+        res: A scalar
+            AP score.
+
+        """
+        relevant = gt_pos.astype(np.bool)
+        rank = rankdata(-pd_scores, "max")[relevant]
+        L = rankdata(-pd_scores[relevant], "max")
+        ans = (L / rank).mean()
+
+        return ans

docs/source/metrics.rst

@@ -31,6 +31,10 @@ Fmeasure (F1)
 -------------------
 .. autoclass:: FMeasure
 
+Mean Average Precision (MAP)
+----------------------------
+.. autoclass:: MAP
+
 Mean Reciprocal Rank (MRR)
 -------------------------------------------
 .. autoclass:: MRR

tests/cornac/metrics/test_ranking.py

@@ -26,14 +26,14 @@
 from cornac.metrics import Recall
 from cornac.metrics import FMeasure
 from cornac.metrics import AUC
+from cornac.metrics import MAP
 
 
 class TestRanking(unittest.TestCase):
-
     def test_ranking_metric(self):
         metric = RankingMetric()
 
-        self.assertEqual(metric.type, 'ranking')
+        self.assertEqual(metric.type, "ranking")
         self.assertIsNone(metric.name)
         self.assertEqual(metric.k, -1)
 
@@ -45,8 +45,8 @@ def test_ranking_metric(self):
     def test_ndcg(self):
         ndcg = NDCG()
 
-        self.assertEqual(ndcg.type, 'ranking')
-        self.assertEqual(ndcg.name, 'NDCG@-1')
+        self.assertEqual(ndcg.type, "ranking")
+        self.assertEqual(ndcg.name, "NDCG@-1")
 
         self.assertEqual(1, ndcg.compute(np.asarray([1]), np.asarray([0])))
 
@@ -59,13 +59,15 @@ def test_ndcg(self):
 
         ground_truth = np.asarray([0, 0, 1])  # [3]
         rec_list = np.asarray([1, 2, 0])  # [2, 3, 1]
-        self.assertEqual(0.63, float('{:.2f}'.format(ndcg_2.compute(ground_truth, rec_list))))
+        self.assertEqual(
+            0.63, float("{:.2f}".format(ndcg_2.compute(ground_truth, rec_list)))
+        )
 
     def test_ncrr(self):
         ncrr = NCRR()
 
-        self.assertEqual(ncrr.type, 'ranking')
-        self.assertEqual(ncrr.name, 'NCRR@-1')
+        self.assertEqual(ncrr.type, "ranking")
+        self.assertEqual(ncrr.name, "NCRR@-1")
 
         self.assertEqual(1, ncrr.compute(np.asarray([1]), np.asarray([0])))
 
@@ -75,7 +77,9 @@ def test_ncrr(self):
 
         ground_truth = np.asarray([1, 0, 1])  # [1, 3]
         rec_list = np.asarray([1, 2, 0])  # [2, 3, 1]
-        self.assertEqual(((1 / 3 + 1 / 2) / (1 + 1 / 2)), ncrr.compute(ground_truth, rec_list))
+        self.assertEqual(
+            ((1 / 3 + 1 / 2) / (1 + 1 / 2)), ncrr.compute(ground_truth, rec_list)
+        )
 
         ncrr_2 = NCRR(k=2)
         self.assertEqual(ncrr_2.k, 2)
@@ -90,18 +94,18 @@ def test_ncrr(self):
 
         ground_truth = np.asarray([1, 1, 1])  # [1, 2, 3]
         rec_list = np.asarray([5, 1, 6])  # [6, 2, 7]
-        self.assertEqual(1./3., ncrr_2.compute(ground_truth, rec_list))
+        self.assertEqual(1.0 / 3.0, ncrr_2.compute(ground_truth, rec_list))
 
         ncrr_3 = NCRR(k=3)
         ground_truth = np.asarray([1, 1])  # [1, 2]
         rec_list = np.asarray([5, 1, 6, 8])  # [6, 2, 7, 9]
-        self.assertEqual(1./3., ncrr_3.compute(ground_truth, rec_list))
+        self.assertEqual(1.0 / 3.0, ncrr_3.compute(ground_truth, rec_list))
 
     def test_mrr(self):
         mrr = MRR()
 
-        self.assertEqual(mrr.type, 'ranking')
-        self.assertEqual(mrr.name, 'MRR')
+        self.assertEqual(mrr.type, "ranking")
+        self.assertEqual(mrr.name, "MRR")
 
         self.assertEqual(1, mrr.compute(np.asarray([1]), np.asarray([0])))
 
@@ -123,7 +127,7 @@ def test_mrr(self):
     def test_measure_at_k(self):
         measure_at_k = MeasureAtK()
 
-        self.assertEqual(measure_at_k.type, 'ranking')
+        self.assertEqual(measure_at_k.type, "ranking")
         assert measure_at_k.name is None
         self.assertEqual(measure_at_k.k, -1)
 
@@ -142,8 +146,8 @@ def test_measure_at_k(self):
     def test_precision(self):
         prec = Precision()
 
-        self.assertEqual(prec.type, 'ranking')
-        self.assertEqual(prec.name, 'Precision@-1')
+        self.assertEqual(prec.type, "ranking")
+        self.assertEqual(prec.name, "Precision@-1")
 
         self.assertEqual(1, prec.compute(np.asarray([1]), np.asarray([0])))
 
@@ -165,8 +169,8 @@ def test_precision(self):
     def test_recall(self):
         rec = Recall()
 
-        self.assertEqual(rec.type, 'ranking')
-        self.assertEqual(rec.name, 'Recall@-1')
+        self.assertEqual(rec.type, "ranking")
+        self.assertEqual(rec.name, "Recall@-1")
 
         self.assertEqual(1, rec.compute(np.asarray([1]), np.asarray([0])))
 
@@ -188,8 +192,8 @@ def test_recall(self):
     def test_f_measure(self):
         f1 = FMeasure()
 
-        self.assertEqual(f1.type, 'ranking')
-        self.assertEqual(f1.name, 'F1@-1')
+        self.assertEqual(f1.type, "ranking")
+        self.assertEqual(f1.name, "F1@-1")
 
         self.assertEqual(1, f1.compute(np.asarray([1]), np.asarray([0])))
 
@@ -215,8 +219,8 @@ def test_f_measure(self):
     def test_auc(self):
         auc = AUC()
 
-        self.assertEqual(auc.type, 'ranking')
-        self.assertEqual(auc.name, 'AUC')
+        self.assertEqual(auc.type, "ranking")
+        self.assertEqual(auc.name, "AUC")
 
         gt_pos = np.array([0, 0, 1, 1])
         pd_scores = np.array([0.1, 0.4, 0.35, 0.8])
@@ -234,6 +238,24 @@ def test_auc(self):
         auc_score = auc.compute(pd_scores, gt_pos, gt_neg)
         self.assertEqual(0.5, auc_score)
 
+    def test_map(self):
+        mAP = MAP()
+
+        self.assertEqual(mAP.type, "ranking")
+        self.assertEqual(mAP.name, "MAP")
+
+        gt_pos = np.array([1, 0, 0])
+        pd_scores = np.array([0.75, 0.5, 1])
+        self.assertEqual(0.5, mAP.compute(pd_scores, gt_pos))
+
+        gt_pos = np.array([0, 0, 1])
+        pd_scores = np.array([1, 0.2, 0.1])
+        self.assertEqual(1 / 3, mAP.compute(pd_scores, gt_pos))
+
+        gt_pos = np.array([0, 1, 0, 1, 0, 1, 0, 0, 0, 0])
+        pd_scores = np.linspace(0.0, 1.0, len(gt_pos))[::-1]
+        self.assertEqual(0.5, mAP.compute(pd_scores, gt_pos))
+
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     unittest.main()