From 71ea276b2e12843c8fbfd07e65e744eccebc5f1a Mon Sep 17 00:00:00 2001
From: Pzzzzz <hello-cd.plus@hotmail.com>
Date: Wed, 28 Dec 2022 17:17:54 +0800
Subject: [PATCH 01/19] init code

---
 mmeval/metrics/__init__.py                    |   3 +-
 mmeval/metrics/instance_seg.py                | 200 ++++++++++
 .../utils/evaluate_semantic_instance.py       | 346 ++++++++++++++++++
 mmeval/metrics/utils/util_3d.py               |  83 +++++
 tests/test_metrics/test_instance_seg.py       |  75 ++++
 5 files changed, 706 insertions(+), 1 deletion(-)
 create mode 100644 mmeval/metrics/instance_seg.py
 create mode 100644 mmeval/metrics/utils/evaluate_semantic_instance.py
 create mode 100644 mmeval/metrics/utils/util_3d.py
 create mode 100644 tests/test_metrics/test_instance_seg.py

diff --git a/mmeval/metrics/__init__.py b/mmeval/metrics/__init__.py
index 31d2b5bd..7b410ed6 100644
--- a/mmeval/metrics/__init__.py
+++ b/mmeval/metrics/__init__.py
@@ -7,6 +7,7 @@
 from .end_point_error import EndPointError
 from .f_metric import F1Metric
 from .hmean_iou import HmeanIoU
+from .instance_seg import InstanceSegMetric
 from .mae import MAE
 from .mean_iou import MeanIoU
 from .mse import MSE
@@ -25,5 +26,5 @@
     'F1Metric', 'HmeanIoU', 'SingleLabelMetric', 'COCODetectionMetric',
     'PCKAccuracy', 'MpiiPCKAccuracy', 'JhmdbPCKAccuracy', 'ProposalRecall',
     'PSNR', 'MAE', 'MSE', 'SSIM', 'SNR', 'MultiLabelMetric',
-    'AveragePrecision', 'AVAMeanAP', 'BLEU'
+    'AveragePrecision', 'AVAMeanAP', 'BLEU', 'InstanceSegMetric'
 ]
diff --git a/mmeval/metrics/instance_seg.py b/mmeval/metrics/instance_seg.py
new file mode 100644
index 00000000..9924acd9
--- /dev/null
+++ b/mmeval/metrics/instance_seg.py
@@ -0,0 +1,200 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+import numpy as np
+from terminaltables import AsciiTable
+from typing import Sequence
+
+from mmeval.core.base_metric import BaseMetric
+from .utils.evaluate_semantic_instance import scannet_eval
+
+
+def aggregate_predictions(masks, labels, scores, valid_class_ids):
+    """Maps predictions to ScanNet evaluator format.
+
+    Args:
+        masks (list[torch.Tensor]): Per scene predicted instance masks.
+        labels (list[torch.Tensor]): Per scene predicted instance labels.
+        scores (list[torch.Tensor]): Per scene predicted instance scores.
+        valid_class_ids (tuple[int]): Ids of valid categories.
+
+    Returns:
+        list[dict]: Per scene aggregated predictions.
+    """
+    infos = []
+    for id, (mask, label, score) in enumerate(zip(masks, labels, scores)):
+        mask = mask.clone().numpy()
+        label = label.clone().numpy()
+        score = score.clone().numpy()
+        info = dict()
+        n_instances = mask.max() + 1
+        for i in range(n_instances):
+            # match pred_instance['filename'] from assign_instances_for_scan
+            file_name = f'{id}_{i}'
+            info[file_name] = dict()
+            info[file_name]['mask'] = (mask == i).astype(np.int)
+            info[file_name]['label_id'] = valid_class_ids[label[i]]
+            info[file_name]['conf'] = score[i]
+        infos.append(info)
+    return infos
+
+
+def rename_gt(gt_semantic_masks, gt_instance_masks, valid_class_ids):
+    """Maps gt instance and semantic masks to instance masks for ScanNet
+    evaluator.
+
+    Args:
+        gt_semantic_masks (list[torch.Tensor]): Per scene gt semantic masks.
+        gt_instance_masks (list[torch.Tensor]): Per scene gt instance masks.
+        valid_class_ids (tuple[int]): Ids of valid categories.
+
+    Returns:
+        list[np.array]: Per scene instance masks.
+    """
+    renamed_instance_masks = []
+    for semantic_mask, instance_mask in zip(gt_semantic_masks,
+                                            gt_instance_masks):
+        semantic_mask = semantic_mask.clone().numpy()
+        instance_mask = instance_mask.clone().numpy()
+        unique = np.unique(instance_mask)
+        assert len(unique) < 1000
+        for i in unique:
+            semantic_instance = semantic_mask[instance_mask == i]
+            semantic_unique = np.unique(semantic_instance)
+            assert len(semantic_unique) == 1
+            if semantic_unique[0] < len(valid_class_ids):
+                instance_mask[
+                    instance_mask ==
+                    i] = 1000 * valid_class_ids[semantic_unique[0]] + i
+        renamed_instance_masks.append(instance_mask)
+    return renamed_instance_masks
+
+
+def instance_seg_eval(gt_semantic_masks,
+                      gt_instance_masks,
+                      pred_instance_masks,
+                      pred_instance_labels,
+                      pred_instance_scores,
+                      valid_class_ids,
+                      class_labels,
+                      options=None):
+    """Instance Segmentation Evaluation.
+
+    Evaluate the result of the instance segmentation.
+
+    Args:
+        gt_semantic_masks (list[torch.Tensor]): Ground truth semantic masks.
+        gt_instance_masks (list[torch.Tensor]): Ground truth instance masks.
+        pred_instance_masks (list[torch.Tensor]): Predicted instance masks.
+        pred_instance_labels (list[torch.Tensor]): Predicted instance labels.
+        pred_instance_scores (list[torch.Tensor]): Predicted instance labels.
+        valid_class_ids (tuple[int]): Ids of valid categories.
+        class_labels (tuple[str]): Names of valid categories.
+        options (dict, optional): Additional options. Keys may contain:
+            `overlaps`, `min_region_sizes`, `distance_threshes`,
+            `distance_confs`. Default: None.
+        logger (logging.Logger | str, optional): The way to print the mAP
+            summary. See `mmdet.utils.print_log()` for details. Default: None.
+
+    Returns:
+        dict[str, float]: Dict of results.
+    """
+    assert len(valid_class_ids) == len(class_labels)
+    id_to_label = {
+        valid_class_ids[i]: class_labels[i]
+        for i in range(len(valid_class_ids))
+    }
+    preds = aggregate_predictions(
+        masks=pred_instance_masks,
+        labels=pred_instance_labels,
+        scores=pred_instance_scores,
+        valid_class_ids=valid_class_ids)
+    gts = rename_gt(gt_semantic_masks, gt_instance_masks, valid_class_ids)
+    metrics = scannet_eval(
+        preds=preds,
+        gts=gts,
+        options=options,
+        valid_class_ids=valid_class_ids,
+        class_labels=class_labels,
+        id_to_label=id_to_label)
+    header = ['classes', 'AP_0.25', 'AP_0.50', 'AP']
+    rows = []
+    for label, data in metrics['classes'].items():
+        aps = [data['ap25%'], data['ap50%'], data['ap']]
+        rows.append([label] + [f'{ap:.4f}' for ap in aps])
+    aps = metrics['all_ap_25%'], metrics['all_ap_50%'], metrics['all_ap']
+    footer = ['Overall'] + [f'{ap:.4f}' for ap in aps]
+    table = AsciiTable([header] + rows + [footer])
+    table.inner_footing_row_border = True
+    return metrics
+
+
+class InstanceSegMetric(BaseMetric):
+    """3D instance segmentation evaluation metric.
+
+    Args:
+        prefix (str): The prefix that will be added in the metric
+            names to disambiguate homonymous metrics of different evaluators.
+            If prefix is not provided in the argument, self.default_prefix
+            will be used instead. Default: None
+    """
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        assert self.dataset_meta is not None
+        self.classes = self.dataset_meta['classes']
+        self.valid_class_ids = self.dataset_meta['seg_valid_class_ids']
+
+    def add(self, predictions: Sequence[dict]) -> None:  # type: ignore # yapf: disable # noqa: E501
+        """Process one batch of data samples and predictions.
+
+        The processed results should be stored in ``self.results``,
+        which will be used to compute the metrics when all batches
+        have been processed.
+        Args:
+            data_batch (dict): A batch of data from the dataloader.
+            data_samples (Sequence[dict]): A batch of outputs from
+                the model.
+        """
+        for data_sample in predictions:
+            pred_3d = data_sample['pred_pts_seg']
+            eval_ann_info = data_sample['eval_ann_info']
+            cpu_pred_3d = dict()
+            for k, v in pred_3d.items():
+                if hasattr(v, 'to'):
+                    cpu_pred_3d[k] = v.to('cpu')
+                else:
+                    cpu_pred_3d[k] = v
+            self._results.append((eval_ann_info, cpu_pred_3d))
+
+    def compute_metrics(self, results: list):
+        """Compute the metrics from processed results.
+
+        Args:
+            results (list): The processed results of each batch.
+        Returns:
+            Dict[str, float]: The computed metrics. The keys are the names of
+            the metrics, and the values are corresponding results.
+        """
+        gt_semantic_masks = []
+        gt_instance_masks = []
+        pred_instance_masks = []
+        pred_instance_labels = []
+        pred_instance_scores = []
+
+        for eval_ann, sinlge_pred_results in results:
+            gt_semantic_masks.append(eval_ann['pts_semantic_mask'])
+            gt_instance_masks.append(eval_ann['pts_instance_mask'])
+            pred_instance_masks.append(
+                sinlge_pred_results['pts_instance_mask'])
+            pred_instance_labels.append(sinlge_pred_results['instance_labels'])
+            pred_instance_scores.append(sinlge_pred_results['instance_scores'])
+
+        ret_dict = instance_seg_eval(
+            gt_semantic_masks,
+            gt_instance_masks,
+            pred_instance_masks,
+            pred_instance_labels,
+            pred_instance_scores,
+            valid_class_ids=self.valid_class_ids,
+            class_labels=self.classes)
+
+        return ret_dict
diff --git a/mmeval/metrics/utils/evaluate_semantic_instance.py b/mmeval/metrics/utils/evaluate_semantic_instance.py
new file mode 100644
index 00000000..8819c4e6
--- /dev/null
+++ b/mmeval/metrics/utils/evaluate_semantic_instance.py
@@ -0,0 +1,346 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+# adapted from https://github.com/ScanNet/ScanNet/blob/master/BenchmarkScripts/3d_evaluation/evaluate_semantic_instance.py # noqa
+import numpy as np
+from copy import deepcopy
+
+from . import util_3d
+
+
+def evaluate_matches(matches, class_labels, options):
+    """Evaluate instance segmentation from matched gt and predicted instances
+    for all scenes.
+
+    Args:
+        matches (dict): Contains gt2pred and pred2gt infos for every scene.
+        class_labels (tuple[str]): Class names.
+        options (dict): ScanNet evaluator options. See get_options.
+
+    Returns:
+        np.array: Average precision scores for all thresholds and categories.
+    """
+    overlaps = options['overlaps']
+    min_region_sizes = [options['min_region_sizes'][0]]
+    dist_threshes = [options['distance_threshes'][0]]
+    dist_confs = [options['distance_confs'][0]]
+
+    # results: class x overlap
+    ap = np.zeros((len(dist_threshes), len(class_labels), len(overlaps)),
+                  np.float)
+    for di, (min_region_size, distance_thresh, distance_conf) in enumerate(
+            zip(min_region_sizes, dist_threshes, dist_confs)):
+        for oi, overlap_th in enumerate(overlaps):
+            pred_visited = {}
+            for m in matches:
+                for label_name in class_labels:
+                    for p in matches[m]['pred'][label_name]:
+                        if 'filename' in p:
+                            pred_visited[p['filename']] = False
+            for li, label_name in enumerate(class_labels):
+                y_true = np.empty(0)
+                y_score = np.empty(0)
+                hard_false_negatives = 0
+                has_gt = False
+                has_pred = False
+                for m in matches:
+                    pred_instances = matches[m]['pred'][label_name]
+                    gt_instances = matches[m]['gt'][label_name]
+                    # filter groups in ground truth
+                    gt_instances = [
+                        gt for gt in gt_instances
+                        if gt['instance_id'] >= 1000 and gt['vert_count'] >=
+                        min_region_size and gt['med_dist'] <= distance_thresh
+                        and gt['dist_conf'] >= distance_conf
+                    ]
+                    if gt_instances:
+                        has_gt = True
+                    if pred_instances:
+                        has_pred = True
+
+                    cur_true = np.ones(len(gt_instances))
+                    cur_score = np.ones(len(gt_instances)) * (-float('inf'))
+                    cur_match = np.zeros(len(gt_instances), dtype=np.bool)
+                    # collect matches
+                    for (gti, gt) in enumerate(gt_instances):
+                        found_match = False
+                        for pred in gt['matched_pred']:
+                            # greedy assignments
+                            if pred_visited[pred['filename']]:
+                                continue
+                            overlap = float(pred['intersection']) / (
+                                gt['vert_count'] + pred['vert_count'] -
+                                pred['intersection'])
+                            if overlap > overlap_th:
+                                confidence = pred['confidence']
+                                # if already have a prediction for this gt,
+                                # the prediction with the lower score is automatically a false positive # noqa
+                                if cur_match[gti]:
+                                    max_score = max(cur_score[gti], confidence)
+                                    min_score = min(cur_score[gti], confidence)
+                                    cur_score[gti] = max_score
+                                    # append false positive
+                                    cur_true = np.append(cur_true, 0)
+                                    cur_score = np.append(cur_score, min_score)
+                                    cur_match = np.append(cur_match, True)
+                                # otherwise set score
+                                else:
+                                    found_match = True
+                                    cur_match[gti] = True
+                                    cur_score[gti] = confidence
+                                    pred_visited[pred['filename']] = True
+                        if not found_match:
+                            hard_false_negatives += 1
+                    # remove non-matched ground truth instances
+                    cur_true = cur_true[cur_match]
+                    cur_score = cur_score[cur_match]
+
+                    # collect non-matched predictions as false positive
+                    for pred in pred_instances:
+                        found_gt = False
+                        for gt in pred['matched_gt']:
+                            overlap = float(gt['intersection']) / (
+                                gt['vert_count'] + pred['vert_count'] -
+                                gt['intersection'])
+                            if overlap > overlap_th:
+                                found_gt = True
+                                break
+                        if not found_gt:
+                            num_ignore = pred['void_intersection']
+                            for gt in pred['matched_gt']:
+                                # group?
+                                if gt['instance_id'] < 1000:
+                                    num_ignore += gt['intersection']
+                                # small ground truth instances
+                                if gt['vert_count'] < min_region_size or gt[
+                                        'med_dist'] > distance_thresh or gt[
+                                            'dist_conf'] < distance_conf:
+                                    num_ignore += gt['intersection']
+                            proportion_ignore = float(
+                                num_ignore) / pred['vert_count']
+                            # if not ignored append false positive
+                            if proportion_ignore <= overlap_th:
+                                cur_true = np.append(cur_true, 0)
+                                confidence = pred['confidence']
+                                cur_score = np.append(cur_score, confidence)
+
+                    # append to overall results
+                    y_true = np.append(y_true, cur_true)
+                    y_score = np.append(y_score, cur_score)
+
+                # compute average precision
+                if has_gt and has_pred:
+                    # compute precision recall curve first
+
+                    # sorting and cumsum
+                    score_arg_sort = np.argsort(y_score)
+                    y_score_sorted = y_score[score_arg_sort]
+                    y_true_sorted = y_true[score_arg_sort]
+                    y_true_sorted_cumsum = np.cumsum(y_true_sorted)
+
+                    # unique thresholds
+                    (thresholds, unique_indices) = np.unique(
+                        y_score_sorted, return_index=True)
+                    num_prec_recall = len(unique_indices) + 1
+
+                    # prepare precision recall
+                    num_examples = len(y_score_sorted)
+                    # follow https://github.com/ScanNet/ScanNet/pull/26 ? # noqa
+                    num_true_examples = y_true_sorted_cumsum[-1] if len(
+                        y_true_sorted_cumsum) > 0 else 0
+                    precision = np.zeros(num_prec_recall)
+                    recall = np.zeros(num_prec_recall)
+
+                    # deal with the first point
+                    y_true_sorted_cumsum = np.append(y_true_sorted_cumsum, 0)
+                    # deal with remaining
+                    for idx_res, idx_scores in enumerate(unique_indices):
+                        cumsum = y_true_sorted_cumsum[idx_scores - 1]
+                        tp = num_true_examples - cumsum
+                        fp = num_examples - idx_scores - tp
+                        fn = cumsum + hard_false_negatives
+                        p = float(tp) / (tp + fp)
+                        r = float(tp) / (tp + fn)
+                        precision[idx_res] = p
+                        recall[idx_res] = r
+
+                    # first point in curve is artificial
+                    precision[-1] = 1.
+                    recall[-1] = 0.
+
+                    # compute average of precision-recall curve
+                    recall_for_conv = np.copy(recall)
+                    recall_for_conv = np.append(recall_for_conv[0],
+                                                recall_for_conv)
+                    recall_for_conv = np.append(recall_for_conv, 0.)
+
+                    stepWidths = np.convolve(recall_for_conv, [-0.5, 0, 0.5],
+                                             'valid')
+                    # integrate is now simply a dot product
+                    ap_current = np.dot(precision, stepWidths)
+
+                elif has_gt:
+                    ap_current = 0.0
+                else:
+                    ap_current = float('nan')
+                ap[di, li, oi] = ap_current
+    return ap
+
+
+def compute_averages(aps, options, class_labels):
+    """Averages AP scores for all categories.
+
+    Args:
+        aps (np.array): AP scores for all thresholds and categories.
+        options (dict): ScanNet evaluator options. See get_options.
+        class_labels (tuple[str]): Class names.
+
+    Returns:
+        dict: Overall and per-category AP scores.
+    """
+    d_inf = 0
+    o50 = np.where(np.isclose(options['overlaps'], 0.5))
+    o25 = np.where(np.isclose(options['overlaps'], 0.25))
+    o_all_but25 = np.where(
+        np.logical_not(np.isclose(options['overlaps'], 0.25)))
+    avg_dict = {}
+    avg_dict['all_ap'] = np.nanmean(aps[d_inf, :, o_all_but25])
+    avg_dict['all_ap_50%'] = np.nanmean(aps[d_inf, :, o50])
+    avg_dict['all_ap_25%'] = np.nanmean(aps[d_inf, :, o25])
+    avg_dict['classes'] = {}
+    for (li, label_name) in enumerate(class_labels):
+        avg_dict['classes'][label_name] = {}
+        avg_dict['classes'][label_name]['ap'] = np.average(aps[d_inf, li,
+                                                               o_all_but25])
+        avg_dict['classes'][label_name]['ap50%'] = np.average(aps[d_inf, li,
+                                                                  o50])
+        avg_dict['classes'][label_name]['ap25%'] = np.average(aps[d_inf, li,
+                                                                  o25])
+    return avg_dict
+
+
+def assign_instances_for_scan(pred_info, gt_ids, options, valid_class_ids,
+                              class_labels, id_to_label):
+    """Assign gt and predicted instances for a single scene.
+
+    Args:
+        pred_info (dict): Predicted masks, labels and scores.
+        gt_ids (np.array): Ground truth instance masks.
+        options (dict): ScanNet evaluator options. See get_options.
+        valid_class_ids (tuple[int]): Ids of valid categories.
+        class_labels (tuple[str]): Class names.
+        id_to_label (dict[int, str]): Mapping of valid class id to class label.
+
+    Returns:
+        dict: Per class assigned gt to predicted instances.
+        dict: Per class assigned predicted to gt instances.
+    """
+    # get gt instances
+    gt_instances = util_3d.get_instances(gt_ids, valid_class_ids, class_labels,
+                                         id_to_label)
+    # associate
+    gt2pred = deepcopy(gt_instances)
+    for label in gt2pred:
+        for gt in gt2pred[label]:
+            gt['matched_pred'] = []
+    pred2gt = {}
+    for label in class_labels:
+        pred2gt[label] = []
+    num_pred_instances = 0
+    # mask of void labels in the ground truth
+    bool_void = np.logical_not(np.in1d(gt_ids // 1000, valid_class_ids))
+    # go through all prediction masks
+    for pred_mask_file in pred_info:
+        label_id = int(pred_info[pred_mask_file]['label_id'])
+        conf = pred_info[pred_mask_file]['conf']
+        if not label_id in id_to_label:  # noqa E713
+            continue
+        label_name = id_to_label[label_id]
+        # read the mask
+        pred_mask = pred_info[pred_mask_file]['mask']
+        if len(pred_mask) != len(gt_ids):
+            raise ValueError('len(pred_mask) != len(gt_ids)')
+        # convert to binary
+        pred_mask = np.not_equal(pred_mask, 0)
+        num = np.count_nonzero(pred_mask)
+        if num < options['min_region_sizes'][0]:
+            continue  # skip if empty
+
+        pred_instance = {}
+        pred_instance['filename'] = pred_mask_file
+        pred_instance['pred_id'] = num_pred_instances
+        pred_instance['label_id'] = label_id
+        pred_instance['vert_count'] = num
+        pred_instance['confidence'] = conf
+        pred_instance['void_intersection'] = np.count_nonzero(
+            np.logical_and(bool_void, pred_mask))
+
+        # matched gt instances
+        matched_gt = []
+        # go through all gt instances with matching label
+        for (gt_num, gt_inst) in enumerate(gt2pred[label_name]):
+            intersection = np.count_nonzero(
+                np.logical_and(gt_ids == gt_inst['instance_id'], pred_mask))
+            if intersection > 0:
+                gt_copy = gt_inst.copy()
+                pred_copy = pred_instance.copy()
+                gt_copy['intersection'] = intersection
+                pred_copy['intersection'] = intersection
+                matched_gt.append(gt_copy)
+                gt2pred[label_name][gt_num]['matched_pred'].append(pred_copy)
+        pred_instance['matched_gt'] = matched_gt
+        num_pred_instances += 1
+        pred2gt[label_name].append(pred_instance)
+
+    return gt2pred, pred2gt
+
+
+def scannet_eval(preds, gts, options, valid_class_ids, class_labels,
+                 id_to_label):
+    """Evaluate instance segmentation in ScanNet protocol.
+
+    Args:
+        preds (list[dict]): Per scene predictions of mask, label and
+            confidence.
+        gts (list[np.array]): Per scene ground truth instance masks.
+        options (dict): ScanNet evaluator options. See get_options.
+        valid_class_ids (tuple[int]): Ids of valid categories.
+        class_labels (tuple[str]): Class names.
+        id_to_label (dict[int, str]): Mapping of valid class id to class label.
+
+    Returns:
+        dict: Overall and per-category AP scores.
+    """
+    options = get_options(options)
+    matches = {}
+    for i, (pred, gt) in enumerate(zip(preds, gts)):
+        matches_key = i
+        # assign gt to predictions
+        gt2pred, pred2gt = assign_instances_for_scan(pred, gt, options,
+                                                     valid_class_ids,
+                                                     class_labels, id_to_label)
+        matches[matches_key] = {}
+        matches[matches_key]['gt'] = gt2pred
+        matches[matches_key]['pred'] = pred2gt
+
+    ap_scores = evaluate_matches(matches, class_labels, options)
+    avgs = compute_averages(ap_scores, options, class_labels)
+    return avgs
+
+
+def get_options(options=None):
+    """Set ScanNet evaluator options.
+
+    Args:
+        options (dict, optional): Not default options. Default: None.
+
+    Returns:
+        dict: Updated options with all 4 keys.
+    """
+    assert options is None or isinstance(options, dict)
+    _options = dict(
+        overlaps=np.append(np.arange(0.5, 0.95, 0.05), 0.25),
+        min_region_sizes=np.array([100]),
+        distance_threshes=np.array([float('inf')]),
+        distance_confs=np.array([-float('inf')]))
+    if options is not None:
+        _options.update(options)
+    return _options
diff --git a/mmeval/metrics/utils/util_3d.py b/mmeval/metrics/utils/util_3d.py
new file mode 100644
index 00000000..fd9291de
--- /dev/null
+++ b/mmeval/metrics/utils/util_3d.py
@@ -0,0 +1,83 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+# adapted from https://github.com/ScanNet/ScanNet/blob/master/BenchmarkScripts/util_3d.py # noqa
+import json
+import numpy as np
+
+
+class Instance:
+    """Single instance for ScanNet evaluator.
+
+    Args:
+        mesh_vert_instances (np.array): Instance ids for each point.
+        instance_id: Id of single instance.
+    """
+    instance_id = 0
+    label_id = 0
+    vert_count = 0
+    med_dist = -1
+    dist_conf = 0.0
+
+    def __init__(self, mesh_vert_instances, instance_id):
+        if instance_id == -1:
+            return
+        self.instance_id = int(instance_id)
+        self.label_id = int(self.get_label_id(instance_id))
+        self.vert_count = int(
+            self.get_instance_verts(mesh_vert_instances, instance_id))
+
+    @staticmethod
+    def get_label_id(instance_id):
+        return int(instance_id // 1000)
+
+    @staticmethod
+    def get_instance_verts(mesh_vert_instances, instance_id):
+        return (mesh_vert_instances == instance_id).sum()
+
+    def to_json(self):
+        return json.dumps(
+            self, default=lambda o: o.__dict__, sort_keys=True, indent=4)
+
+    def to_dict(self):
+        dict = {}
+        dict['instance_id'] = self.instance_id
+        dict['label_id'] = self.label_id
+        dict['vert_count'] = self.vert_count
+        dict['med_dist'] = self.med_dist
+        dict['dist_conf'] = self.dist_conf
+        return dict
+
+    def from_json(self, data):
+        self.instance_id = int(data['instance_id'])
+        self.label_id = int(data['label_id'])
+        self.vert_count = int(data['vert_count'])
+        if 'med_dist' in data:
+            self.med_dist = float(data['med_dist'])
+            self.dist_conf = float(data['dist_conf'])
+
+    def __str__(self):
+        return '(' + str(self.instance_id) + ')'
+
+
+def get_instances(ids, class_ids, class_labels, id2label):
+    """Transform gt instance mask to Instance objects.
+
+    Args:
+        ids (np.array): Instance ids for each point.
+        class_ids: (tuple[int]): Ids of valid categories.
+        class_labels (tuple[str]): Class names.
+        id2label: (dict[int, str]): Mapping of valid class id to class label.
+
+    Returns:
+        dict [str, list]: Instance objects grouped by class label.
+    """
+    instances = {}
+    for label in class_labels:
+        instances[label] = []
+    instance_ids = np.unique(ids)
+    for id in instance_ids:
+        if id == 0:
+            continue
+        inst = Instance(ids, id)
+        if inst.label_id in class_ids:
+            instances[id2label[inst.label_id]].append(inst.to_dict())
+    return instances
diff --git a/tests/test_metrics/test_instance_seg.py b/tests/test_metrics/test_instance_seg.py
new file mode 100644
index 00000000..016c50c4
--- /dev/null
+++ b/tests/test_metrics/test_instance_seg.py
@@ -0,0 +1,75 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+import numpy as np
+import torch
+
+from mmeval.metrics.instance_seg import instance_seg_eval
+
+
+def test_instance_seg_eval():
+    valid_class_ids = (3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 14, 16, 24, 28, 33, 34,
+                       36, 39)
+    class_labels = ('cabinet', 'bed', 'chair', 'sofa', 'table', 'door',
+                    'window', 'bookshelf', 'picture', 'counter', 'desk',
+                    'curtain', 'refrigerator', 'showercurtrain', 'toilet',
+                    'sink', 'bathtub', 'garbagebin')
+    n_points_list = [3300, 3000]
+    gt_labels_list = [[0, 0, 0, 0, 0, 0, 14, 14, 2, 1],
+                      [13, 13, 2, 1, 3, 3, 0, 0, 0]]
+    gt_instance_masks = []
+    gt_semantic_masks = []
+    pred_instance_masks = []
+    pred_instance_labels = []
+    pred_instance_scores = []
+    for n_points, gt_labels in zip(n_points_list, gt_labels_list):
+        gt_instance_mask = np.ones(n_points, dtype=np.int) * -1
+        gt_semantic_mask = np.ones(n_points, dtype=np.int) * -1
+        pred_instance_mask = np.ones(n_points, dtype=np.int) * -1
+        labels = []
+        scores = []
+        for i, gt_label in enumerate(gt_labels):
+            begin = i * 300
+            end = begin + 300
+            gt_instance_mask[begin:end] = i
+            gt_semantic_mask[begin:end] = gt_label
+            pred_instance_mask[begin:end] = i
+            labels.append(gt_label)
+            scores.append(.99)
+        gt_instance_masks.append(torch.tensor(gt_instance_mask))
+        gt_semantic_masks.append(torch.tensor(gt_semantic_mask))
+        pred_instance_masks.append(torch.tensor(pred_instance_mask))
+        pred_instance_labels.append(torch.tensor(labels))
+        pred_instance_scores.append(torch.tensor(scores))
+
+    ret_value = instance_seg_eval(
+        gt_semantic_masks=gt_semantic_masks,
+        gt_instance_masks=gt_instance_masks,
+        pred_instance_masks=pred_instance_masks,
+        pred_instance_labels=pred_instance_labels,
+        pred_instance_scores=pred_instance_scores,
+        valid_class_ids=valid_class_ids,
+        class_labels=class_labels)
+    for label in [
+            'cabinet', 'bed', 'chair', 'sofa', 'showercurtrain', 'toilet'
+    ]:
+        metrics = ret_value['classes'][label]
+        assert metrics['ap'] == 1.0
+        assert metrics['ap50%'] == 1.0
+        assert metrics['ap25%'] == 1.0
+
+    pred_instance_masks[1][2240:2700] = -1
+    pred_instance_masks[0][2700:3000] = 8
+    pred_instance_labels[0][9] = 2
+    ret_value = instance_seg_eval(
+        gt_semantic_masks=gt_semantic_masks,
+        gt_instance_masks=gt_instance_masks,
+        pred_instance_masks=pred_instance_masks,
+        pred_instance_labels=pred_instance_labels,
+        pred_instance_scores=pred_instance_scores,
+        valid_class_ids=valid_class_ids,
+        class_labels=class_labels)
+    assert abs(ret_value['classes']['cabinet']['ap50%'] - 0.72916) < 0.01
+    assert abs(ret_value['classes']['cabinet']['ap25%'] - 0.88888) < 0.01
+    assert abs(ret_value['classes']['bed']['ap50%'] - 0.5) < 0.01
+    assert abs(ret_value['classes']['bed']['ap25%'] - 0.5) < 0.01
+    assert abs(ret_value['classes']['chair']['ap50%'] - 0.375) < 0.01
+    assert abs(ret_value['classes']['chair']['ap25%'] - 1.0) < 0.01

From 7bad06eaae0ffba6a9aa883551d7b8b51580eff1 Mon Sep 17 00:00:00 2001
From: Pzzzzz <hello-cd.plus@hotmail.com>
Date: Wed, 28 Dec 2022 19:21:21 +0800
Subject: [PATCH 02/19] finished v0.1

---
 mmeval/metrics/instance_seg.py                |  77 ++++-----
 mmeval/metrics/utils/__init__.py              |   4 +-
 .../utils/evaluate_semantic_instance.py       |   4 +-
 tests/test_metrics/test_instance_seg.py       | 162 ++++++++++--------
 4 files changed, 132 insertions(+), 115 deletions(-)

diff --git a/mmeval/metrics/instance_seg.py b/mmeval/metrics/instance_seg.py
index 9924acd9..12ebaf00 100644
--- a/mmeval/metrics/instance_seg.py
+++ b/mmeval/metrics/instance_seg.py
@@ -1,19 +1,20 @@
 # Copyright (c) OpenMMLab. All rights reserved.
 import numpy as np
+from copy import deepcopy
 from terminaltables import AsciiTable
-from typing import Sequence
+from typing import Dict, List, Sequence
 
 from mmeval.core.base_metric import BaseMetric
-from .utils.evaluate_semantic_instance import scannet_eval
+from mmeval.metrics.utils import scannet_eval
 
 
 def aggregate_predictions(masks, labels, scores, valid_class_ids):
     """Maps predictions to ScanNet evaluator format.
 
     Args:
-        masks (list[torch.Tensor]): Per scene predicted instance masks.
-        labels (list[torch.Tensor]): Per scene predicted instance labels.
-        scores (list[torch.Tensor]): Per scene predicted instance scores.
+        masks (list[numpy.ndarray]): Per scene predicted instance masks.
+        labels (list[numpy.ndarray]): Per scene predicted instance labels.
+        scores (list[numpy.ndarray]): Per scene predicted instance scores.
         valid_class_ids (tuple[int]): Ids of valid categories.
 
     Returns:
@@ -21,16 +22,13 @@ def aggregate_predictions(masks, labels, scores, valid_class_ids):
     """
     infos = []
     for id, (mask, label, score) in enumerate(zip(masks, labels, scores)):
-        mask = mask.clone().numpy()
-        label = label.clone().numpy()
-        score = score.clone().numpy()
         info = dict()
         n_instances = mask.max() + 1
         for i in range(n_instances):
             # match pred_instance['filename'] from assign_instances_for_scan
             file_name = f'{id}_{i}'
             info[file_name] = dict()
-            info[file_name]['mask'] = (mask == i).astype(np.int)
+            info[file_name]['mask'] = (mask == i).astype(int)
             info[file_name]['label_id'] = valid_class_ids[label[i]]
             info[file_name]['conf'] = score[i]
         infos.append(info)
@@ -42,8 +40,8 @@ def rename_gt(gt_semantic_masks, gt_instance_masks, valid_class_ids):
     evaluator.
 
     Args:
-        gt_semantic_masks (list[torch.Tensor]): Per scene gt semantic masks.
-        gt_instance_masks (list[torch.Tensor]): Per scene gt instance masks.
+        gt_semantic_masks (list[numpy.ndarray]): Per scene gt semantic masks.
+        gt_instance_masks (list[numpy.ndarray]): Per scene gt instance masks.
         valid_class_ids (tuple[int]): Ids of valid categories.
 
     Returns:
@@ -52,8 +50,6 @@ def rename_gt(gt_semantic_masks, gt_instance_masks, valid_class_ids):
     renamed_instance_masks = []
     for semantic_mask, instance_mask in zip(gt_semantic_masks,
                                             gt_instance_masks):
-        semantic_mask = semantic_mask.clone().numpy()
-        instance_mask = instance_mask.clone().numpy()
         unique = np.unique(instance_mask)
         assert len(unique) < 1000
         for i in unique:
@@ -81,11 +77,11 @@ def instance_seg_eval(gt_semantic_masks,
     Evaluate the result of the instance segmentation.
 
     Args:
-        gt_semantic_masks (list[torch.Tensor]): Ground truth semantic masks.
-        gt_instance_masks (list[torch.Tensor]): Ground truth instance masks.
-        pred_instance_masks (list[torch.Tensor]): Predicted instance masks.
-        pred_instance_labels (list[torch.Tensor]): Predicted instance labels.
-        pred_instance_scores (list[torch.Tensor]): Predicted instance labels.
+        gt_semantic_masks (list[numpy.ndarray]): Ground truth semantic masks.
+        gt_instance_masks (list[numpy.ndarray]): Ground truth instance masks.
+        pred_instance_masks (list[numpy.ndarray]): Predicted instance masks.
+        pred_instance_labels (list[numpy.ndarray]): Predicted instance labels.
+        pred_instance_scores (list[numpy.ndarray]): Predicted instance labels.
         valid_class_ids (tuple[int]): Ids of valid categories.
         class_labels (tuple[str]): Names of valid categories.
         options (dict, optional): Additional options. Keys may contain:
@@ -143,29 +139,29 @@ def __init__(self, **kwargs):
         self.classes = self.dataset_meta['classes']
         self.valid_class_ids = self.dataset_meta['seg_valid_class_ids']
 
-    def add(self, predictions: Sequence[dict]) -> None:  # type: ignore # yapf: disable # noqa: E501
+    def add(self, predictions: Sequence[Dict], groundtruths: Sequence[Dict]) -> None:  # type: ignore # yapf: disable # noqa: E501
         """Process one batch of data samples and predictions.
 
         The processed results should be stored in ``self.results``,
         which will be used to compute the metrics when all batches
         have been processed.
         Args:
-            data_batch (dict): A batch of data from the dataloader.
-            data_samples (Sequence[dict]): A batch of outputs from
-                the model.
+            predictions (Sequence[Dict]): A sequence of dict. Each dict
+                representing a detection result, with the following keys:
+
+            - pts_instance_mask(numpy.ndarray): Haha
+            - instance_labels(numpy.ndarray): Haha
+            - instance_scores(numpy.ndarray): Haha
+            groundtruths (Sequence[Dict]): A sequence of dict. Each dict
+                represents a groundtruths for an image, with the following
+                keys:
+            - pts_instance_mask(numpy.ndarray): Haha
+            - pts_semantic_mask(numpy.ndarray): Haha
         """
-        for data_sample in predictions:
-            pred_3d = data_sample['pred_pts_seg']
-            eval_ann_info = data_sample['eval_ann_info']
-            cpu_pred_3d = dict()
-            for k, v in pred_3d.items():
-                if hasattr(v, 'to'):
-                    cpu_pred_3d[k] = v.to('cpu')
-                else:
-                    cpu_pred_3d[k] = v
-            self._results.append((eval_ann_info, cpu_pred_3d))
-
-    def compute_metrics(self, results: list):
+        for prediction, groundtruth in zip(predictions, groundtruths):
+            self._results.append((deepcopy(prediction), deepcopy(groundtruth)))
+
+    def compute_metric(self, results: List[List[Dict]]) -> Dict[str, float]:
         """Compute the metrics from processed results.
 
         Args:
@@ -180,13 +176,12 @@ def compute_metrics(self, results: list):
         pred_instance_labels = []
         pred_instance_scores = []
 
-        for eval_ann, sinlge_pred_results in results:
-            gt_semantic_masks.append(eval_ann['pts_semantic_mask'])
-            gt_instance_masks.append(eval_ann['pts_instance_mask'])
-            pred_instance_masks.append(
-                sinlge_pred_results['pts_instance_mask'])
-            pred_instance_labels.append(sinlge_pred_results['instance_labels'])
-            pred_instance_scores.append(sinlge_pred_results['instance_scores'])
+        for result_pred, result_gt in results:
+            gt_semantic_masks.append(result_gt['pts_semantic_mask'])
+            gt_instance_masks.append(result_gt['pts_instance_mask'])
+            pred_instance_masks.append(result_pred['pts_instance_mask'])
+            pred_instance_labels.append(result_pred['instance_labels'])
+            pred_instance_scores.append(result_pred['instance_scores'])
 
         ret_dict = instance_seg_eval(
             gt_semantic_masks,
diff --git a/mmeval/metrics/utils/__init__.py b/mmeval/metrics/utils/__init__.py
index 200575c0..9feb715d 100644
--- a/mmeval/metrics/utils/__init__.py
+++ b/mmeval/metrics/utils/__init__.py
@@ -1,5 +1,6 @@
 # Copyright (c) OpenMMLab. All rights reserved.
 from .bbox_overlaps import calculate_bboxes_area, calculate_overlaps
+from .evaluate_semantic_instance import scannet_eval
 from .image_transforms import reorder_and_crop
 from .keypoint import calc_distances, distance_acc
 from .polygon import (poly2shapely, poly_intersection, poly_iou,
@@ -8,5 +9,6 @@
 __all__ = [
     'poly2shapely', 'polys2shapely', 'poly_union', 'poly_intersection',
     'poly_make_valid', 'poly_iou', 'calc_distances', 'distance_acc',
-    'calculate_overlaps', 'calculate_bboxes_area', 'reorder_and_crop'
+    'calculate_overlaps', 'calculate_bboxes_area', 'reorder_and_crop',
+    'scannet_eval'
 ]
diff --git a/mmeval/metrics/utils/evaluate_semantic_instance.py b/mmeval/metrics/utils/evaluate_semantic_instance.py
index 8819c4e6..397f863d 100644
--- a/mmeval/metrics/utils/evaluate_semantic_instance.py
+++ b/mmeval/metrics/utils/evaluate_semantic_instance.py
@@ -25,7 +25,7 @@ def evaluate_matches(matches, class_labels, options):
 
     # results: class x overlap
     ap = np.zeros((len(dist_threshes), len(class_labels), len(overlaps)),
-                  np.float)
+                  float)
     for di, (min_region_size, distance_thresh, distance_conf) in enumerate(
             zip(min_region_sizes, dist_threshes, dist_confs)):
         for oi, overlap_th in enumerate(overlaps):
@@ -58,7 +58,7 @@ def evaluate_matches(matches, class_labels, options):
 
                     cur_true = np.ones(len(gt_instances))
                     cur_score = np.ones(len(gt_instances)) * (-float('inf'))
-                    cur_match = np.zeros(len(gt_instances), dtype=np.bool)
+                    cur_match = np.zeros(len(gt_instances), dtype=bool)
                     # collect matches
                     for (gti, gt) in enumerate(gt_instances):
                         found_match = False
diff --git a/tests/test_metrics/test_instance_seg.py b/tests/test_metrics/test_instance_seg.py
index 016c50c4..7cde6017 100644
--- a/tests/test_metrics/test_instance_seg.py
+++ b/tests/test_metrics/test_instance_seg.py
@@ -1,75 +1,95 @@
 # Copyright (c) OpenMMLab. All rights reserved.
 import numpy as np
 import torch
+import unittest
 
-from mmeval.metrics.instance_seg import instance_seg_eval
-
-
-def test_instance_seg_eval():
-    valid_class_ids = (3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 14, 16, 24, 28, 33, 34,
-                       36, 39)
-    class_labels = ('cabinet', 'bed', 'chair', 'sofa', 'table', 'door',
-                    'window', 'bookshelf', 'picture', 'counter', 'desk',
-                    'curtain', 'refrigerator', 'showercurtrain', 'toilet',
-                    'sink', 'bathtub', 'garbagebin')
-    n_points_list = [3300, 3000]
-    gt_labels_list = [[0, 0, 0, 0, 0, 0, 14, 14, 2, 1],
-                      [13, 13, 2, 1, 3, 3, 0, 0, 0]]
-    gt_instance_masks = []
-    gt_semantic_masks = []
-    pred_instance_masks = []
-    pred_instance_labels = []
-    pred_instance_scores = []
-    for n_points, gt_labels in zip(n_points_list, gt_labels_list):
-        gt_instance_mask = np.ones(n_points, dtype=np.int) * -1
-        gt_semantic_mask = np.ones(n_points, dtype=np.int) * -1
-        pred_instance_mask = np.ones(n_points, dtype=np.int) * -1
-        labels = []
-        scores = []
-        for i, gt_label in enumerate(gt_labels):
-            begin = i * 300
-            end = begin + 300
-            gt_instance_mask[begin:end] = i
-            gt_semantic_mask[begin:end] = gt_label
-            pred_instance_mask[begin:end] = i
-            labels.append(gt_label)
-            scores.append(.99)
-        gt_instance_masks.append(torch.tensor(gt_instance_mask))
-        gt_semantic_masks.append(torch.tensor(gt_semantic_mask))
-        pred_instance_masks.append(torch.tensor(pred_instance_mask))
-        pred_instance_labels.append(torch.tensor(labels))
-        pred_instance_scores.append(torch.tensor(scores))
-
-    ret_value = instance_seg_eval(
-        gt_semantic_masks=gt_semantic_masks,
-        gt_instance_masks=gt_instance_masks,
-        pred_instance_masks=pred_instance_masks,
-        pred_instance_labels=pred_instance_labels,
-        pred_instance_scores=pred_instance_scores,
-        valid_class_ids=valid_class_ids,
-        class_labels=class_labels)
-    for label in [
-            'cabinet', 'bed', 'chair', 'sofa', 'showercurtrain', 'toilet'
-    ]:
-        metrics = ret_value['classes'][label]
-        assert metrics['ap'] == 1.0
-        assert metrics['ap50%'] == 1.0
-        assert metrics['ap25%'] == 1.0
-
-    pred_instance_masks[1][2240:2700] = -1
-    pred_instance_masks[0][2700:3000] = 8
-    pred_instance_labels[0][9] = 2
-    ret_value = instance_seg_eval(
-        gt_semantic_masks=gt_semantic_masks,
-        gt_instance_masks=gt_instance_masks,
-        pred_instance_masks=pred_instance_masks,
-        pred_instance_labels=pred_instance_labels,
-        pred_instance_scores=pred_instance_scores,
-        valid_class_ids=valid_class_ids,
-        class_labels=class_labels)
-    assert abs(ret_value['classes']['cabinet']['ap50%'] - 0.72916) < 0.01
-    assert abs(ret_value['classes']['cabinet']['ap25%'] - 0.88888) < 0.01
-    assert abs(ret_value['classes']['bed']['ap50%'] - 0.5) < 0.01
-    assert abs(ret_value['classes']['bed']['ap25%'] - 0.5) < 0.01
-    assert abs(ret_value['classes']['chair']['ap50%'] - 0.375) < 0.01
-    assert abs(ret_value['classes']['chair']['ap25%'] - 1.0) < 0.01
+from mmeval.metrics import InstanceSegMetric
+
+
+class TestInstanceSegMetric(unittest.TestCase):
+
+    def _demo_mm_model_output(self):
+        """Create a superset of inputs needed to run test or train batches."""
+
+        n_points_list = [3300, 3000]
+        gt_labels_list = [[0, 0, 0, 0, 0, 0, 14, 14, 2, 1],
+                          [13, 13, 2, 1, 3, 3, 0, 0, 0]]
+
+        predictions = []
+        groundtruths = []
+
+        for n_points, gt_labels in zip(n_points_list, gt_labels_list):
+            gt_instance_mask = np.ones(n_points, dtype=int) * -1
+            gt_semantic_mask = np.ones(n_points, dtype=int) * -1
+            for i, gt_label in enumerate(gt_labels):
+                begin = i * 300
+                end = begin + 300
+                gt_instance_mask[begin:end] = i
+                gt_semantic_mask[begin:end] = gt_label
+
+            ann_info_data = dict()
+            ann_info_data['pts_instance_mask'] = torch.tensor(gt_instance_mask)
+            ann_info_data['pts_semantic_mask'] = torch.tensor(gt_semantic_mask)
+
+            results_dict = dict()
+            pred_instance_mask = np.ones(n_points, dtype=int) * -1
+            labels = []
+            scores = []
+            for i, gt_label in enumerate(gt_labels):
+                begin = i * 300
+                end = begin + 300
+                pred_instance_mask[begin:end] = i
+                labels.append(gt_label)
+                scores.append(.99)
+
+            results_dict['pts_instance_mask'] = torch.tensor(
+                pred_instance_mask)
+            results_dict['instance_labels'] = torch.tensor(labels)
+            results_dict['instance_scores'] = torch.tensor(scores)
+
+            predictions.append(results_dict)
+            groundtruths.append(ann_info_data)
+
+        return predictions, groundtruths
+
+    def test_evaluate(self):
+        predictions, groundtruths = self._demo_mm_model_output()
+        predictions = [{k: v.clone().numpy()
+                        for k, v in i.items()} for i in predictions]
+        groundtruths = [{k: v.clone().numpy()
+                         for k, v in i.items()} for i in groundtruths]
+        seg_valid_class_ids = (3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 14, 16, 24, 28,
+                               33, 34, 36, 39)
+        class_labels = ('cabinet', 'bed', 'chair', 'sofa', 'table', 'door',
+                        'window', 'bookshelf', 'picture', 'counter', 'desk',
+                        'curtain', 'refrigerator', 'showercurtrain', 'toilet',
+                        'sink', 'bathtub', 'garbagebin')
+        dataset_meta = dict(
+            seg_valid_class_ids=seg_valid_class_ids, classes=class_labels)
+        instance_seg_metric = InstanceSegMetric(dataset_meta=dataset_meta)
+        res = instance_seg_metric(predictions, groundtruths)
+        self.assertIsInstance(res, dict)
+        for label in [
+                'cabinet', 'bed', 'chair', 'sofa', 'showercurtrain', 'toilet'
+        ]:
+            metrics = res['classes'][label]
+            assert metrics['ap'] == 1.0
+            assert metrics['ap50%'] == 1.0
+            assert metrics['ap25%'] == 1.0
+
+        predictions[1]['pts_instance_mask'][2240:2700] = -1
+        predictions[0]['pts_instance_mask'][2700:3000] = 8
+        predictions[0]['instance_labels'][9] = 2
+
+        instance_seg_metric.reset()
+
+        res = instance_seg_metric(predictions, groundtruths)
+
+        print(res)
+
+        assert abs(res['classes']['cabinet']['ap50%'] - 0.72916) < 0.01
+        assert abs(res['classes']['cabinet']['ap25%'] - 0.88888) < 0.01
+        assert abs(res['classes']['bed']['ap50%'] - 0.5) < 0.01
+        assert abs(res['classes']['bed']['ap25%'] - 0.5) < 0.01
+        assert abs(res['classes']['chair']['ap50%'] - 0.375) < 0.01
+        assert abs(res['classes']['chair']['ap25%'] - 1.0) < 0.01

From 08fb0f2b935ba299c754d4b5a05308ab4d16b403 Mon Sep 17 00:00:00 2001
From: Pzzzzz <hello-cd.plus@hotmail.com>
Date: Wed, 28 Dec 2022 19:29:59 +0800
Subject: [PATCH 03/19] remove print

---
 tests/test_metrics/test_instance_seg.py | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/tests/test_metrics/test_instance_seg.py b/tests/test_metrics/test_instance_seg.py
index 7cde6017..eaf22072 100644
--- a/tests/test_metrics/test_instance_seg.py
+++ b/tests/test_metrics/test_instance_seg.py
@@ -81,12 +81,8 @@ def test_evaluate(self):
         predictions[0]['pts_instance_mask'][2700:3000] = 8
         predictions[0]['instance_labels'][9] = 2
 
-        instance_seg_metric.reset()
-
         res = instance_seg_metric(predictions, groundtruths)
 
-        print(res)
-
         assert abs(res['classes']['cabinet']['ap50%'] - 0.72916) < 0.01
         assert abs(res['classes']['cabinet']['ap25%'] - 0.88888) < 0.01
         assert abs(res['classes']['bed']['ap50%'] - 0.5) < 0.01

From ebbd5b2b9b47868c55cc6c7da29be107ef549f2e Mon Sep 17 00:00:00 2001
From: Pzzzzz <hello-cd.plus@hotmail.com>
Date: Wed, 28 Dec 2022 22:32:55 +0800
Subject: [PATCH 04/19] update doc

---
 mmeval/metrics/instance_seg.py | 69 ++++++++++++++++++++++++++++++++--
 1 file changed, 65 insertions(+), 4 deletions(-)

diff --git a/mmeval/metrics/instance_seg.py b/mmeval/metrics/instance_seg.py
index 12ebaf00..421a7335 100644
--- a/mmeval/metrics/instance_seg.py
+++ b/mmeval/metrics/instance_seg.py
@@ -127,10 +127,71 @@ class InstanceSegMetric(BaseMetric):
     """3D instance segmentation evaluation metric.
 
     Args:
-        prefix (str): The prefix that will be added in the metric
-            names to disambiguate homonymous metrics of different evaluators.
-            If prefix is not provided in the argument, self.default_prefix
-            will be used instead. Default: None
+        dataset_meta (dict): Provide dataset meta information.
+
+    Example:
+        >>> import numpy as np
+        >>> from mmeval import InstanceSegMetric
+        >>> seg_valid_class_ids = (3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 14, 16, 24,
+        >>>                 28, 33, 34, 36, 39)
+        >>> class_labels = ('cabinet', 'bed', 'chair', 'sofa', 'table', 'door',
+        ...         'window', 'bookshelf', 'picture', 'counter', 'desk',
+        ...         'curtain', 'refrigerator', 'showercurtrain', 'toilet',
+        ...         'sink', 'bathtub', 'garbagebin')
+        >>> dataset_meta = dict(
+        ...     seg_valid_class_ids=seg_valid_class_ids, classes=class_labels)
+        >>>
+        >>> def _demo_mm_model_output(self):
+        ...     n_points_list = [3300, 3000]
+        ...     gt_labels_list = [[0, 0, 0, 0, 0, 0, 14, 14, 2, 1],
+        ...                       [13, 13, 2, 1, 3, 3, 0, 0, 0]]
+        ...
+        ...     predictions = []
+        ...     groundtruths = []
+        ...
+        ...     for n_points, gt_labels in zip(n_points_list, gt_labels_list):
+        ...         gt_instance_mask = np.ones(n_points, dtype=int) * -1
+        ...         gt_semantic_mask = np.ones(n_points, dtype=int) * -1
+        ...         for i, gt_label in enumerate(gt_labels):
+        ...             begin = i * 300
+        ...             end = begin + 300
+        ...             gt_instance_mask[begin:end] = i
+        ...             gt_semantic_mask[begin:end] = gt_label
+        ...
+        ...         ann_info_data = dict()
+        ...         ann_info_data['pts_instance_mask'] = torch.tensor(
+        ...             gt_instance_mask)
+        ...         ann_info_data['pts_semantic_mask'] = torch.tensor(
+        ...             gt_semantic_mask)
+        ...
+        ...         results_dict = dict()
+        ...         pred_instance_mask = np.ones(n_points, dtype=int) * -1
+        ...         labels = []
+        ...         scores = []
+        ...         for i, gt_label in enumerate(gt_labels):
+        ...             begin = i * 300
+        ...             end = begin + 300
+        ...             pred_instance_mask[begin:end] = i
+        ...             labels.append(gt_label)
+        ...             scores.append(.99)
+        ...
+        ...         results_dict['pts_instance_mask'] = torch.tensor(
+        ...             pred_instance_mask)
+        ...         results_dict['instance_labels'] = torch.tensor(labels)
+        ...         results_dict['instance_scores'] = torch.tensor(scores)
+        ...
+        ...         predictions.append(results_dict)
+        ...         groundtruths.append(ann_info_data)
+        ...
+        ...     return predictions, groundtruths
+        >>>
+        >>> instance_seg_metric = InstanceSegMetric(dataset_meta=dataset_meta)
+        >>> res = instance_seg_metric(predictions, groundtruths)
+        >>> res
+        {'all_ap': 1.0, 'all_ap_50%': 1.0, 'all_ap_25%': 1.0,
+         'classes':
+           {'cabinet': {'ap': 1.0, 'ap50%': 1.0, 'ap25%': 1.0},
+            'bed': {'ap': 1.0, 'ap50%': 1.0, 'ap25%': 1.0}, ...}}
     """
 
     def __init__(self, **kwargs):

From 02c723f1615d444901fc86281cf276398a31d063 Mon Sep 17 00:00:00 2001
From: Pzzzzz <hello-cd.plus@hotmail.com>
Date: Wed, 28 Dec 2022 22:35:05 +0800
Subject: [PATCH 05/19] update doc

---
 mmeval/metrics/instance_seg.py | 27 +++++++++++++++++++++++----
 1 file changed, 23 insertions(+), 4 deletions(-)

diff --git a/mmeval/metrics/instance_seg.py b/mmeval/metrics/instance_seg.py
index 421a7335..9ef0c1c4 100644
--- a/mmeval/metrics/instance_seg.py
+++ b/mmeval/metrics/instance_seg.py
@@ -188,10 +188,29 @@ class InstanceSegMetric(BaseMetric):
         >>> instance_seg_metric = InstanceSegMetric(dataset_meta=dataset_meta)
         >>> res = instance_seg_metric(predictions, groundtruths)
         >>> res
-        {'all_ap': 1.0, 'all_ap_50%': 1.0, 'all_ap_25%': 1.0,
-         'classes':
-           {'cabinet': {'ap': 1.0, 'ap50%': 1.0, 'ap25%': 1.0},
-            'bed': {'ap': 1.0, 'ap50%': 1.0, 'ap25%': 1.0}, ...}}
+        {
+        'all_ap': 1.0,
+        'all_ap_50%': 1.0,
+        'all_ap_25%': 1.0,
+        'classes': {
+            'cabinet': {
+                'ap': 1.0,
+                'ap50%': 1.0,
+                'ap25%': 1.0
+            },
+            'bed': {
+                'ap': 1.0,
+                'ap50%': 1.0,
+                'ap25%': 1.0
+            },
+            'chair': {
+                'ap': 1.0,
+                'ap50%': 1.0,
+                'ap25%': 1.0
+            },
+            ...
+        }
+    }
     """
 
     def __init__(self, **kwargs):

From 6733185ecc6bc1fedf4a008ac9c01df256d956a5 Mon Sep 17 00:00:00 2001
From: Pzzzzz <hello-cd.plus@hotmail.com>
Date: Thu, 29 Dec 2022 16:02:42 +0800
Subject: [PATCH 06/19] rename InstanceSegMetric

---
 mmeval/metrics/__init__.py              | 4 ++--
 mmeval/metrics/instance_seg.py          | 2 +-
 tests/test_metrics/test_instance_seg.py | 4 ++--
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/mmeval/metrics/__init__.py b/mmeval/metrics/__init__.py
index 7b410ed6..941ca3f8 100644
--- a/mmeval/metrics/__init__.py
+++ b/mmeval/metrics/__init__.py
@@ -7,7 +7,7 @@
 from .end_point_error import EndPointError
 from .f_metric import F1Metric
 from .hmean_iou import HmeanIoU
-from .instance_seg import InstanceSegMetric
+from .instance_seg import InstanceSeg
 from .mae import MAE
 from .mean_iou import MeanIoU
 from .mse import MSE
@@ -26,5 +26,5 @@
     'F1Metric', 'HmeanIoU', 'SingleLabelMetric', 'COCODetectionMetric',
     'PCKAccuracy', 'MpiiPCKAccuracy', 'JhmdbPCKAccuracy', 'ProposalRecall',
     'PSNR', 'MAE', 'MSE', 'SSIM', 'SNR', 'MultiLabelMetric',
-    'AveragePrecision', 'AVAMeanAP', 'BLEU', 'InstanceSegMetric'
+    'AveragePrecision', 'AVAMeanAP', 'BLEU', 'InstanceSeg'
 ]
diff --git a/mmeval/metrics/instance_seg.py b/mmeval/metrics/instance_seg.py
index 9ef0c1c4..4d6c88ae 100644
--- a/mmeval/metrics/instance_seg.py
+++ b/mmeval/metrics/instance_seg.py
@@ -123,7 +123,7 @@ def instance_seg_eval(gt_semantic_masks,
     return metrics
 
 
-class InstanceSegMetric(BaseMetric):
+class InstanceSeg(BaseMetric):
     """3D instance segmentation evaluation metric.
 
     Args:
diff --git a/tests/test_metrics/test_instance_seg.py b/tests/test_metrics/test_instance_seg.py
index eaf22072..ca8eaa66 100644
--- a/tests/test_metrics/test_instance_seg.py
+++ b/tests/test_metrics/test_instance_seg.py
@@ -3,7 +3,7 @@
 import torch
 import unittest
 
-from mmeval.metrics import InstanceSegMetric
+from mmeval.metrics import InstanceSeg
 
 
 class TestInstanceSegMetric(unittest.TestCase):
@@ -66,7 +66,7 @@ def test_evaluate(self):
                         'sink', 'bathtub', 'garbagebin')
         dataset_meta = dict(
             seg_valid_class_ids=seg_valid_class_ids, classes=class_labels)
-        instance_seg_metric = InstanceSegMetric(dataset_meta=dataset_meta)
+        instance_seg_metric = InstanceSeg(dataset_meta=dataset_meta)
         res = instance_seg_metric(predictions, groundtruths)
         self.assertIsInstance(res, dict)
         for label in [

From 64fd213083e6f2bf2ce8a2470554ece42c373032 Mon Sep 17 00:00:00 2001
From: Pzzzzz <hello-cd.plus@hotmail.com>
Date: Thu, 29 Dec 2022 16:33:22 +0800
Subject: [PATCH 07/19] update doc

---
 mmeval/metrics/instance_seg.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/mmeval/metrics/instance_seg.py b/mmeval/metrics/instance_seg.py
index 4d6c88ae..ffc02e9e 100644
--- a/mmeval/metrics/instance_seg.py
+++ b/mmeval/metrics/instance_seg.py
@@ -81,7 +81,7 @@ def instance_seg_eval(gt_semantic_masks,
         gt_instance_masks (list[numpy.ndarray]): Ground truth instance masks.
         pred_instance_masks (list[numpy.ndarray]): Predicted instance masks.
         pred_instance_labels (list[numpy.ndarray]): Predicted instance labels.
-        pred_instance_scores (list[numpy.ndarray]): Predicted instance labels.
+        pred_instance_scores (list[numpy.ndarray]): Predicted instance scores.
         valid_class_ids (tuple[int]): Ids of valid categories.
         class_labels (tuple[str]): Names of valid categories.
         options (dict, optional): Additional options. Keys may contain:
@@ -229,14 +229,14 @@ def add(self, predictions: Sequence[Dict], groundtruths: Sequence[Dict]) -> None
             predictions (Sequence[Dict]): A sequence of dict. Each dict
                 representing a detection result, with the following keys:
 
-            - pts_instance_mask(numpy.ndarray): Haha
-            - instance_labels(numpy.ndarray): Haha
-            - instance_scores(numpy.ndarray): Haha
+            - pts_instance_mask(numpy.ndarray): Predicted instance masks.
+            - instance_labels(numpy.ndarray): Predicted instance labels.
+            - instance_scores(numpy.ndarray): Predicted instance scores.
             groundtruths (Sequence[Dict]): A sequence of dict. Each dict
                 represents a groundtruths for an image, with the following
                 keys:
-            - pts_instance_mask(numpy.ndarray): Haha
-            - pts_semantic_mask(numpy.ndarray): Haha
+            - pts_instance_mask(numpy.ndarray): Ground truth instance masks.
+            - pts_semantic_mask(numpy.ndarray): Ground truth semantic masks.
         """
         for prediction, groundtruth in zip(predictions, groundtruths):
             self._results.append((deepcopy(prediction), deepcopy(groundtruth)))

From c64423928f89f9a41eef87ee6b6d07405539d9ca Mon Sep 17 00:00:00 2001
From: Pzzzzz <hello-cd.plus@hotmail.com>
Date: Fri, 30 Dec 2022 16:37:34 +0800
Subject: [PATCH 08/19] remove tables

---
 mmeval/metrics/instance_seg.py | 10 ----------
 1 file changed, 10 deletions(-)

diff --git a/mmeval/metrics/instance_seg.py b/mmeval/metrics/instance_seg.py
index ffc02e9e..25c3ed49 100644
--- a/mmeval/metrics/instance_seg.py
+++ b/mmeval/metrics/instance_seg.py
@@ -1,7 +1,6 @@
 # Copyright (c) OpenMMLab. All rights reserved.
 import numpy as np
 from copy import deepcopy
-from terminaltables import AsciiTable
 from typing import Dict, List, Sequence
 
 from mmeval.core.base_metric import BaseMetric
@@ -111,15 +110,6 @@ def instance_seg_eval(gt_semantic_masks,
         valid_class_ids=valid_class_ids,
         class_labels=class_labels,
         id_to_label=id_to_label)
-    header = ['classes', 'AP_0.25', 'AP_0.50', 'AP']
-    rows = []
-    for label, data in metrics['classes'].items():
-        aps = [data['ap25%'], data['ap50%'], data['ap']]
-        rows.append([label] + [f'{ap:.4f}' for ap in aps])
-    aps = metrics['all_ap_25%'], metrics['all_ap_50%'], metrics['all_ap']
-    footer = ['Overall'] + [f'{ap:.4f}' for ap in aps]
-    table = AsciiTable([header] + rows + [footer])
-    table.inner_footing_row_border = True
     return metrics
 
 

From 6a255380f4b663d21116322f632ed2d2030dc253 Mon Sep 17 00:00:00 2001
From: Pzzzzz <hello-cd.plus@hotmail.com>
Date: Tue, 3 Jan 2023 11:00:41 +0800
Subject: [PATCH 09/19] move some files to _vendor

---
 mmeval/metrics/_vendor/scannet/README.md                      | 2 ++
 mmeval/metrics/_vendor/scannet/__init__.py                    | 4 ++++
 .../{utils => _vendor/scannet}/evaluate_semantic_instance.py  | 0
 mmeval/metrics/{utils => _vendor/scannet}/util_3d.py          | 0
 mmeval/metrics/instance_seg.py                                | 2 +-
 mmeval/metrics/utils/__init__.py                              | 4 +---
 6 files changed, 8 insertions(+), 4 deletions(-)
 create mode 100644 mmeval/metrics/_vendor/scannet/README.md
 create mode 100644 mmeval/metrics/_vendor/scannet/__init__.py
 rename mmeval/metrics/{utils => _vendor/scannet}/evaluate_semantic_instance.py (100%)
 rename mmeval/metrics/{utils => _vendor/scannet}/util_3d.py (100%)

diff --git a/mmeval/metrics/_vendor/scannet/README.md b/mmeval/metrics/_vendor/scannet/README.md
new file mode 100644
index 00000000..46134b19
--- /dev/null
+++ b/mmeval/metrics/_vendor/scannet/README.md
@@ -0,0 +1,2 @@
+The code under this folder is from the official [ScanNet repo](https://github.com/ScanNet/ScanNet).
+Some unused codes are removed to minimize the length of codes added.
diff --git a/mmeval/metrics/_vendor/scannet/__init__.py b/mmeval/metrics/_vendor/scannet/__init__.py
new file mode 100644
index 00000000..812196a3
--- /dev/null
+++ b/mmeval/metrics/_vendor/scannet/__init__.py
@@ -0,0 +1,4 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+from .evaluate_semantic_instance import scannet_eval
+
+__all__ = ['scannet_eval']
diff --git a/mmeval/metrics/utils/evaluate_semantic_instance.py b/mmeval/metrics/_vendor/scannet/evaluate_semantic_instance.py
similarity index 100%
rename from mmeval/metrics/utils/evaluate_semantic_instance.py
rename to mmeval/metrics/_vendor/scannet/evaluate_semantic_instance.py
diff --git a/mmeval/metrics/utils/util_3d.py b/mmeval/metrics/_vendor/scannet/util_3d.py
similarity index 100%
rename from mmeval/metrics/utils/util_3d.py
rename to mmeval/metrics/_vendor/scannet/util_3d.py
diff --git a/mmeval/metrics/instance_seg.py b/mmeval/metrics/instance_seg.py
index 25c3ed49..fa979e23 100644
--- a/mmeval/metrics/instance_seg.py
+++ b/mmeval/metrics/instance_seg.py
@@ -4,7 +4,7 @@
 from typing import Dict, List, Sequence
 
 from mmeval.core.base_metric import BaseMetric
-from mmeval.metrics.utils import scannet_eval
+from mmeval.metrics._vendor.scannet import scannet_eval
 
 
 def aggregate_predictions(masks, labels, scores, valid_class_ids):
diff --git a/mmeval/metrics/utils/__init__.py b/mmeval/metrics/utils/__init__.py
index 9feb715d..200575c0 100644
--- a/mmeval/metrics/utils/__init__.py
+++ b/mmeval/metrics/utils/__init__.py
@@ -1,6 +1,5 @@
 # Copyright (c) OpenMMLab. All rights reserved.
 from .bbox_overlaps import calculate_bboxes_area, calculate_overlaps
-from .evaluate_semantic_instance import scannet_eval
 from .image_transforms import reorder_and_crop
 from .keypoint import calc_distances, distance_acc
 from .polygon import (poly2shapely, poly_intersection, poly_iou,
@@ -9,6 +8,5 @@
 __all__ = [
     'poly2shapely', 'polys2shapely', 'poly_union', 'poly_intersection',
     'poly_make_valid', 'poly_iou', 'calc_distances', 'distance_acc',
-    'calculate_overlaps', 'calculate_bboxes_area', 'reorder_and_crop',
-    'scannet_eval'
+    'calculate_overlaps', 'calculate_bboxes_area', 'reorder_and_crop'
 ]

From 95d75cf5631f35b8a7d024e8af30811da62b6219 Mon Sep 17 00:00:00 2001
From: Pzzzzz <31173671+Pzzzzz5142@users.noreply.github.com>
Date: Thu, 5 Jan 2023 17:42:48 +0800
Subject: [PATCH 10/19] Update mmeval/metrics/instance_seg.py

Co-authored-by: yancong <32220263+ice-tong@users.noreply.github.com>
---
 mmeval/metrics/instance_seg.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/mmeval/metrics/instance_seg.py b/mmeval/metrics/instance_seg.py
index fa979e23..0fdae89f 100644
--- a/mmeval/metrics/instance_seg.py
+++ b/mmeval/metrics/instance_seg.py
@@ -236,6 +236,7 @@ def compute_metric(self, results: List[List[Dict]]) -> Dict[str, float]:
 
         Args:
             results (list): The processed results of each batch.
+ 
         Returns:
             Dict[str, float]: The computed metrics. The keys are the names of
             the metrics, and the values are corresponding results.

From cfc234b7c7bf2fa26c67fe1a69f4f1fed2ec8e18 Mon Sep 17 00:00:00 2001
From: Pzzzzz <31173671+Pzzzzz5142@users.noreply.github.com>
Date: Thu, 5 Jan 2023 17:43:38 +0800
Subject: [PATCH 11/19] Update mmeval/metrics/instance_seg.py

Co-authored-by: yancong <32220263+ice-tong@users.noreply.github.com>
---
 mmeval/metrics/instance_seg.py | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/mmeval/metrics/instance_seg.py b/mmeval/metrics/instance_seg.py
index 0fdae89f..68261489 100644
--- a/mmeval/metrics/instance_seg.py
+++ b/mmeval/metrics/instance_seg.py
@@ -215,18 +215,21 @@ def add(self, predictions: Sequence[Dict], groundtruths: Sequence[Dict]) -> None
         The processed results should be stored in ``self.results``,
         which will be used to compute the metrics when all batches
         have been processed.
+        
         Args:
             predictions (Sequence[Dict]): A sequence of dict. Each dict
                 representing a detection result, with the following keys:
 
-            - pts_instance_mask(numpy.ndarray): Predicted instance masks.
-            - instance_labels(numpy.ndarray): Predicted instance labels.
-            - instance_scores(numpy.ndarray): Predicted instance scores.
+                - pts_instance_mask(numpy.ndarray): Predicted instance masks.
+                - instance_labels(numpy.ndarray): Predicted instance labels.
+                - instance_scores(numpy.ndarray): Predicted instance scores.
+            
             groundtruths (Sequence[Dict]): A sequence of dict. Each dict
                 represents a groundtruths for an image, with the following
                 keys:
-            - pts_instance_mask(numpy.ndarray): Ground truth instance masks.
-            - pts_semantic_mask(numpy.ndarray): Ground truth semantic masks.
+            
+                - pts_instance_mask(numpy.ndarray): Ground truth instance masks.
+                - pts_semantic_mask(numpy.ndarray): Ground truth semantic masks.
         """
         for prediction, groundtruth in zip(predictions, groundtruths):
             self._results.append((deepcopy(prediction), deepcopy(groundtruth)))

From 50a257dbd8ac296dc2c644ca50e540a9d2a32a94 Mon Sep 17 00:00:00 2001
From: Pzzzzz <hello-cd.plus@hotmail.com>
Date: Thu, 5 Jan 2023 18:21:59 +0800
Subject: [PATCH 12/19] update according to code review

---
 mmeval/metrics/instance_seg.py          | 263 +++++++++++++-----------
 tests/test_metrics/test_instance_seg.py | 168 +++++++--------
 2 files changed, 225 insertions(+), 206 deletions(-)

diff --git a/mmeval/metrics/instance_seg.py b/mmeval/metrics/instance_seg.py
index 68261489..5451f198 100644
--- a/mmeval/metrics/instance_seg.py
+++ b/mmeval/metrics/instance_seg.py
@@ -1,123 +1,22 @@
 # Copyright (c) OpenMMLab. All rights reserved.
 import numpy as np
+import warnings
 from copy import deepcopy
-from typing import Dict, List, Sequence
+from typing import Dict, List, Optional, Sequence
 
 from mmeval.core.base_metric import BaseMetric
 from mmeval.metrics._vendor.scannet import scannet_eval
 
 
-def aggregate_predictions(masks, labels, scores, valid_class_ids):
-    """Maps predictions to ScanNet evaluator format.
-
-    Args:
-        masks (list[numpy.ndarray]): Per scene predicted instance masks.
-        labels (list[numpy.ndarray]): Per scene predicted instance labels.
-        scores (list[numpy.ndarray]): Per scene predicted instance scores.
-        valid_class_ids (tuple[int]): Ids of valid categories.
-
-    Returns:
-        list[dict]: Per scene aggregated predictions.
-    """
-    infos = []
-    for id, (mask, label, score) in enumerate(zip(masks, labels, scores)):
-        info = dict()
-        n_instances = mask.max() + 1
-        for i in range(n_instances):
-            # match pred_instance['filename'] from assign_instances_for_scan
-            file_name = f'{id}_{i}'
-            info[file_name] = dict()
-            info[file_name]['mask'] = (mask == i).astype(int)
-            info[file_name]['label_id'] = valid_class_ids[label[i]]
-            info[file_name]['conf'] = score[i]
-        infos.append(info)
-    return infos
-
-
-def rename_gt(gt_semantic_masks, gt_instance_masks, valid_class_ids):
-    """Maps gt instance and semantic masks to instance masks for ScanNet
-    evaluator.
-
-    Args:
-        gt_semantic_masks (list[numpy.ndarray]): Per scene gt semantic masks.
-        gt_instance_masks (list[numpy.ndarray]): Per scene gt instance masks.
-        valid_class_ids (tuple[int]): Ids of valid categories.
-
-    Returns:
-        list[np.array]: Per scene instance masks.
-    """
-    renamed_instance_masks = []
-    for semantic_mask, instance_mask in zip(gt_semantic_masks,
-                                            gt_instance_masks):
-        unique = np.unique(instance_mask)
-        assert len(unique) < 1000
-        for i in unique:
-            semantic_instance = semantic_mask[instance_mask == i]
-            semantic_unique = np.unique(semantic_instance)
-            assert len(semantic_unique) == 1
-            if semantic_unique[0] < len(valid_class_ids):
-                instance_mask[
-                    instance_mask ==
-                    i] = 1000 * valid_class_ids[semantic_unique[0]] + i
-        renamed_instance_masks.append(instance_mask)
-    return renamed_instance_masks
-
-
-def instance_seg_eval(gt_semantic_masks,
-                      gt_instance_masks,
-                      pred_instance_masks,
-                      pred_instance_labels,
-                      pred_instance_scores,
-                      valid_class_ids,
-                      class_labels,
-                      options=None):
-    """Instance Segmentation Evaluation.
-
-    Evaluate the result of the instance segmentation.
-
-    Args:
-        gt_semantic_masks (list[numpy.ndarray]): Ground truth semantic masks.
-        gt_instance_masks (list[numpy.ndarray]): Ground truth instance masks.
-        pred_instance_masks (list[numpy.ndarray]): Predicted instance masks.
-        pred_instance_labels (list[numpy.ndarray]): Predicted instance labels.
-        pred_instance_scores (list[numpy.ndarray]): Predicted instance scores.
-        valid_class_ids (tuple[int]): Ids of valid categories.
-        class_labels (tuple[str]): Names of valid categories.
-        options (dict, optional): Additional options. Keys may contain:
-            `overlaps`, `min_region_sizes`, `distance_threshes`,
-            `distance_confs`. Default: None.
-        logger (logging.Logger | str, optional): The way to print the mAP
-            summary. See `mmdet.utils.print_log()` for details. Default: None.
-
-    Returns:
-        dict[str, float]: Dict of results.
-    """
-    assert len(valid_class_ids) == len(class_labels)
-    id_to_label = {
-        valid_class_ids[i]: class_labels[i]
-        for i in range(len(valid_class_ids))
-    }
-    preds = aggregate_predictions(
-        masks=pred_instance_masks,
-        labels=pred_instance_labels,
-        scores=pred_instance_scores,
-        valid_class_ids=valid_class_ids)
-    gts = rename_gt(gt_semantic_masks, gt_instance_masks, valid_class_ids)
-    metrics = scannet_eval(
-        preds=preds,
-        gts=gts,
-        options=options,
-        valid_class_ids=valid_class_ids,
-        class_labels=class_labels,
-        id_to_label=id_to_label)
-    return metrics
-
-
 class InstanceSeg(BaseMetric):
     """3D instance segmentation evaluation metric.
 
     Args:
-        dataset_meta (dict): Provide dataset meta information.
+        dataset_meta (dict, optional): Provide dataset meta information.
+        classes (List[str], optional): Provide dataset classes information as
+            an alternative to dataset_meta.
+        valid_class_ids (List[int], optional): Provide dataset valid class ids
+            information as an alternative to dataset_meta.
 
     Example:
         >>> import numpy as np
@@ -203,11 +102,26 @@ class InstanceSeg(BaseMetric):
     }
     """
 
-    def __init__(self, **kwargs):
+    def __init__(self,
+                 classes: Optional[List[str]] = None,
+                 valid_class_ids: Optional[List[int]] = None,
+                 **kwargs):
         super().__init__(**kwargs)
-        assert self.dataset_meta is not None
-        self.classes = self.dataset_meta['classes']
-        self.valid_class_ids = self.dataset_meta['seg_valid_class_ids']
+        assert (self.dataset_meta
+                is not None) or (classes is not None
+                                 and valid_class_ids is not None)
+        if self.dataset_meta is not None:
+            if classes is not None or valid_class_ids is not None:
+                warnings.warn('`classes` and `valid_class_ids` are ignored.')
+            self.classes = self.dataset_meta['classes']
+            self.valid_class_ids = self.dataset_meta['seg_valid_class_ids']
+        else:
+            if not isinstance(classes, list):
+                raise TypeError('`classes` should be List[str]')
+            if not isinstance(valid_class_ids, list):
+                raise TypeError('`valid_class_ids` should be List[int]')
+            self.classes = classes
+            self.valid_class_ids = valid_class_ids
 
     def add(self, predictions: Sequence[Dict], groundtruths: Sequence[Dict]) -> None:  # type: ignore # yapf: disable # noqa: E501
         """Process one batch of data samples and predictions.
@@ -215,21 +129,21 @@ def add(self, predictions: Sequence[Dict], groundtruths: Sequence[Dict]) -> None
         The processed results should be stored in ``self.results``,
         which will be used to compute the metrics when all batches
         have been processed.
-        
+
         Args:
             predictions (Sequence[Dict]): A sequence of dict. Each dict
                 representing a detection result, with the following keys:
 
-                - pts_instance_mask(numpy.ndarray): Predicted instance masks.
-                - instance_labels(numpy.ndarray): Predicted instance labels.
-                - instance_scores(numpy.ndarray): Predicted instance scores.
-            
+                - pts_instance_mask(np.ndarray): Predicted instance masks.
+                - instance_labels(np.ndarray): Predicted instance labels.
+                - instance_scores(np.ndarray): Predicted instance scores.
+
             groundtruths (Sequence[Dict]): A sequence of dict. Each dict
                 represents a groundtruths for an image, with the following
                 keys:
-            
-                - pts_instance_mask(numpy.ndarray): Ground truth instance masks.
-                - pts_semantic_mask(numpy.ndarray): Ground truth semantic masks.
+
+                - pts_instance_mask(np.ndarray): Ground truth instance masks.
+                - pts_semantic_mask(np.ndarray): Ground truth semantic masks.
         """
         for prediction, groundtruth in zip(predictions, groundtruths):
             self._results.append((deepcopy(prediction), deepcopy(groundtruth)))
@@ -239,7 +153,7 @@ def compute_metric(self, results: List[List[Dict]]) -> Dict[str, float]:
 
         Args:
             results (list): The processed results of each batch.
- 
+
         Returns:
             Dict[str, float]: The computed metrics. The keys are the names of
             the metrics, and the values are corresponding results.
@@ -257,7 +171,7 @@ def compute_metric(self, results: List[List[Dict]]) -> Dict[str, float]:
             pred_instance_labels.append(result_pred['instance_labels'])
             pred_instance_scores.append(result_pred['instance_scores'])
 
-        ret_dict = instance_seg_eval(
+        ret_dict = self.instance_seg_eval(
             gt_semantic_masks,
             gt_instance_masks,
             pred_instance_masks,
@@ -267,3 +181,108 @@ def compute_metric(self, results: List[List[Dict]]) -> Dict[str, float]:
             class_labels=self.classes)
 
         return ret_dict
+
+    def aggregate_predictions(self, masks, labels, scores, valid_class_ids):
+        """Maps predictions to ScanNet evaluator format.
+
+        Args:
+            masks (list[np.ndarray]): Per scene predicted instance masks.
+            labels (list[np.ndarray]): Per scene predicted instance labels.
+            scores (list[np.ndarray]): Per scene predicted instance scores.
+            valid_class_ids (tuple[int]): Ids of valid categories.
+
+        Returns:
+            list[dict]: Per scene aggregated predictions.
+        """
+        infos = []
+        for id, (mask, label, score) in enumerate(zip(masks, labels, scores)):
+            info = dict()
+            n_instances = mask.max() + 1
+            for i in range(n_instances):
+                file_name = f'{id}_{i}'
+                info[file_name] = dict()
+                info[file_name]['mask'] = (mask == i).astype(int)
+                info[file_name]['label_id'] = valid_class_ids[label[i]]
+                info[file_name]['conf'] = score[i]
+            infos.append(info)
+        return infos
+
+    def rename_gt(self, gt_semantic_masks, gt_instance_masks, valid_class_ids):
+        """Maps gt instance and semantic masks to instance masks for ScanNet
+        evaluator.
+
+        Args:
+            gt_semantic_masks (list[np.ndarray]): Per scene gt semantic masks.
+            gt_instance_masks (list[np.ndarray]): Per scene gt instance masks.
+            valid_class_ids (tuple[int]): Ids of valid categories.
+
+        Returns:
+            list[np.array]: Per scene instance masks.
+        """
+        renamed_instance_masks = []
+        for semantic_mask, instance_mask in zip(gt_semantic_masks,
+                                                gt_instance_masks):
+            unique = np.unique(instance_mask)
+            assert len(unique) < 1000
+            for i in unique:
+                semantic_instance = semantic_mask[instance_mask == i]
+                semantic_unique = np.unique(semantic_instance)
+                assert len(semantic_unique) == 1
+                if semantic_unique[0] < len(valid_class_ids):
+                    instance_mask[
+                        instance_mask ==
+                        i] = 1000 * valid_class_ids[semantic_unique[0]] + i
+            renamed_instance_masks.append(instance_mask)
+        return renamed_instance_masks
+
+    def instance_seg_eval(self,
+                          gt_semantic_masks,
+                          gt_instance_masks,
+                          pred_instance_masks,
+                          pred_instance_labels,
+                          pred_instance_scores,
+                          valid_class_ids,
+                          class_labels,
+                          options=None):
+        """Instance Segmentation Evaluation.
+
+        Evaluate the result of the instance segmentation.
+
+        Args:
+            gt_semantic_masks (list[np.ndarray]): Ground truth semantic masks.
+            gt_instance_masks (list[np.ndarray]): Ground truth instance masks.
+            pred_instance_masks (list[np.ndarray]): Predicted instance masks.
+            pred_instance_labels (list[np.ndarray]): Predicted instance labels.
+            pred_instance_scores (list[np.ndarray]): Predicted instance scores.
+            valid_class_ids (tuple[int]): Ids of valid categories.
+            class_labels (tuple[str]): Names of valid categories.
+            options (dict, optional): Additional options. Keys may contain:
+                `overlaps`, `min_region_sizes`, `distance_threshes`,
+                `distance_confs`. Default: None.
+            logger (logging.Logger | str, optional): The way to print the
+                mAP summary. See `mmdet.utils.print_log()` for details.
+                Default: None.
+
+        Returns:
+            dict[str, float]: Dict of results.
+        """
+        assert len(valid_class_ids) == len(class_labels)
+        id_to_label = {
+            valid_class_ids[i]: class_labels[i]
+            for i in range(len(valid_class_ids))
+        }
+        preds = self.aggregate_predictions(
+            masks=pred_instance_masks,
+            labels=pred_instance_labels,
+            scores=pred_instance_scores,
+            valid_class_ids=valid_class_ids)
+        gts = self.rename_gt(gt_semantic_masks, gt_instance_masks,
+                             valid_class_ids)
+        metrics = scannet_eval(
+            preds=preds,
+            gts=gts,
+            options=options,
+            valid_class_ids=valid_class_ids,
+            class_labels=class_labels,
+            id_to_label=id_to_label)
+        return metrics
diff --git a/tests/test_metrics/test_instance_seg.py b/tests/test_metrics/test_instance_seg.py
index ca8eaa66..f66407b9 100644
--- a/tests/test_metrics/test_instance_seg.py
+++ b/tests/test_metrics/test_instance_seg.py
@@ -1,91 +1,91 @@
 # Copyright (c) OpenMMLab. All rights reserved.
 import numpy as np
+import pytest
 import torch
-import unittest
 
 from mmeval.metrics import InstanceSeg
 
 
-class TestInstanceSegMetric(unittest.TestCase):
-
-    def _demo_mm_model_output(self):
-        """Create a superset of inputs needed to run test or train batches."""
-
-        n_points_list = [3300, 3000]
-        gt_labels_list = [[0, 0, 0, 0, 0, 0, 14, 14, 2, 1],
-                          [13, 13, 2, 1, 3, 3, 0, 0, 0]]
-
-        predictions = []
-        groundtruths = []
-
-        for n_points, gt_labels in zip(n_points_list, gt_labels_list):
-            gt_instance_mask = np.ones(n_points, dtype=int) * -1
-            gt_semantic_mask = np.ones(n_points, dtype=int) * -1
-            for i, gt_label in enumerate(gt_labels):
-                begin = i * 300
-                end = begin + 300
-                gt_instance_mask[begin:end] = i
-                gt_semantic_mask[begin:end] = gt_label
-
-            ann_info_data = dict()
-            ann_info_data['pts_instance_mask'] = torch.tensor(gt_instance_mask)
-            ann_info_data['pts_semantic_mask'] = torch.tensor(gt_semantic_mask)
-
-            results_dict = dict()
-            pred_instance_mask = np.ones(n_points, dtype=int) * -1
-            labels = []
-            scores = []
-            for i, gt_label in enumerate(gt_labels):
-                begin = i * 300
-                end = begin + 300
-                pred_instance_mask[begin:end] = i
-                labels.append(gt_label)
-                scores.append(.99)
-
-            results_dict['pts_instance_mask'] = torch.tensor(
-                pred_instance_mask)
-            results_dict['instance_labels'] = torch.tensor(labels)
-            results_dict['instance_scores'] = torch.tensor(scores)
-
-            predictions.append(results_dict)
-            groundtruths.append(ann_info_data)
-
-        return predictions, groundtruths
-
-    def test_evaluate(self):
-        predictions, groundtruths = self._demo_mm_model_output()
-        predictions = [{k: v.clone().numpy()
-                        for k, v in i.items()} for i in predictions]
-        groundtruths = [{k: v.clone().numpy()
-                         for k, v in i.items()} for i in groundtruths]
-        seg_valid_class_ids = (3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 14, 16, 24, 28,
-                               33, 34, 36, 39)
-        class_labels = ('cabinet', 'bed', 'chair', 'sofa', 'table', 'door',
-                        'window', 'bookshelf', 'picture', 'counter', 'desk',
-                        'curtain', 'refrigerator', 'showercurtrain', 'toilet',
-                        'sink', 'bathtub', 'garbagebin')
-        dataset_meta = dict(
-            seg_valid_class_ids=seg_valid_class_ids, classes=class_labels)
-        instance_seg_metric = InstanceSeg(dataset_meta=dataset_meta)
-        res = instance_seg_metric(predictions, groundtruths)
-        self.assertIsInstance(res, dict)
-        for label in [
-                'cabinet', 'bed', 'chair', 'sofa', 'showercurtrain', 'toilet'
-        ]:
-            metrics = res['classes'][label]
-            assert metrics['ap'] == 1.0
-            assert metrics['ap50%'] == 1.0
-            assert metrics['ap25%'] == 1.0
-
-        predictions[1]['pts_instance_mask'][2240:2700] = -1
-        predictions[0]['pts_instance_mask'][2700:3000] = 8
-        predictions[0]['instance_labels'][9] = 2
-
-        res = instance_seg_metric(predictions, groundtruths)
-
-        assert abs(res['classes']['cabinet']['ap50%'] - 0.72916) < 0.01
-        assert abs(res['classes']['cabinet']['ap25%'] - 0.88888) < 0.01
-        assert abs(res['classes']['bed']['ap50%'] - 0.5) < 0.01
-        assert abs(res['classes']['bed']['ap25%'] - 0.5) < 0.01
-        assert abs(res['classes']['chair']['ap50%'] - 0.375) < 0.01
-        assert abs(res['classes']['chair']['ap25%'] - 1.0) < 0.01
+def _demo_mm_model_output():
+    """Create a superset of inputs needed to run test or train batches."""
+    n_points_list = [3300, 3000]
+    gt_labels_list = [[0, 0, 0, 0, 0, 0, 14, 14, 2, 1],
+                      [13, 13, 2, 1, 3, 3, 0, 0, 0]]
+    predictions = []
+    groundtruths = []
+    for n_points, gt_labels in zip(n_points_list, gt_labels_list):
+        gt_instance_mask = np.ones(n_points, dtype=int) * -1
+        gt_semantic_mask = np.ones(n_points, dtype=int) * -1
+        for i, gt_label in enumerate(gt_labels):
+            begin = i * 300
+            end = begin + 300
+            gt_instance_mask[begin:end] = i
+            gt_semantic_mask[begin:end] = gt_label
+        ann_info_data = dict()
+        ann_info_data['pts_instance_mask'] = torch.tensor(gt_instance_mask)
+        ann_info_data['pts_semantic_mask'] = torch.tensor(gt_semantic_mask)
+        results_dict = dict()
+        pred_instance_mask = np.ones(n_points, dtype=int) * -1
+        labels = []
+        scores = []
+        for i, gt_label in enumerate(gt_labels):
+            begin = i * 300
+            end = begin + 300
+            pred_instance_mask[begin:end] = i
+            labels.append(gt_label)
+            scores.append(.99)
+        results_dict['pts_instance_mask'] = torch.tensor(pred_instance_mask)
+        results_dict['instance_labels'] = torch.tensor(labels)
+        results_dict['instance_scores'] = torch.tensor(scores)
+        predictions.append(results_dict)
+        groundtruths.append(ann_info_data)
+    return predictions, groundtruths
+
+
+def test_metric_invalid_usage():
+    with pytest.raises(AssertionError):
+        InstanceSeg()
+
+    with pytest.raises(AssertionError):
+        InstanceSeg(
+            valid_class_ids=(3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 14, 16, 24, 28,
+                             33, 34, 36, 39))
+
+    with pytest.raises(TypeError):
+        InstanceSeg(classes='a', valid_class_ids='b')
+
+
+def test_evaluate():
+    predictions, groundtruths = _demo_mm_model_output()
+    predictions = [{k: v.clone().numpy()
+                    for k, v in i.items()} for i in predictions]
+    groundtruths = [{k: v.clone().numpy()
+                     for k, v in i.items()} for i in groundtruths]
+    seg_valid_class_ids = (3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 14, 16, 24, 28, 33,
+                           34, 36, 39)
+    class_labels = ('cabinet', 'bed', 'chair', 'sofa', 'table', 'door',
+                    'window', 'bookshelf', 'picture', 'counter', 'desk',
+                    'curtain', 'refrigerator', 'showercurtrain', 'toilet',
+                    'sink', 'bathtub', 'garbagebin')
+    dataset_meta = dict(
+        seg_valid_class_ids=seg_valid_class_ids, classes=class_labels)
+    instance_seg_metric = InstanceSeg(dataset_meta=dataset_meta)
+    res = instance_seg_metric(predictions, groundtruths)
+    assert isinstance(res, dict)
+    for label in [
+            'cabinet', 'bed', 'chair', 'sofa', 'showercurtrain', 'toilet'
+    ]:
+        metrics = res['classes'][label]
+        assert metrics['ap'] == 1.0
+        assert metrics['ap50%'] == 1.0
+        assert metrics['ap25%'] == 1.0
+    predictions[1]['pts_instance_mask'][2240:2700] = -1
+    predictions[0]['pts_instance_mask'][2700:3000] = 8
+    predictions[0]['instance_labels'][9] = 2
+    res = instance_seg_metric(predictions, groundtruths)
+    assert abs(res['classes']['cabinet']['ap50%'] - 0.72916) < 0.01
+    assert abs(res['classes']['cabinet']['ap25%'] - 0.88888) < 0.01
+    assert abs(res['classes']['bed']['ap50%'] - 0.5) < 0.01
+    assert abs(res['classes']['bed']['ap25%'] - 0.5) < 0.01
+    assert abs(res['classes']['chair']['ap50%'] - 0.375) < 0.01
+    assert abs(res['classes']['chair']['ap25%'] - 1.0) < 0.01

From 22142a65012aff082d5e27bb72b299954e34f908 Mon Sep 17 00:00:00 2001
From: Pzzzzz <31173671+Pzzzzz5142@users.noreply.github.com>
Date: Sun, 8 Jan 2023 17:18:16 +0800
Subject: [PATCH 13/19] Update metrics.rst

---
 docs/zh_cn/api/metrics.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/zh_cn/api/metrics.rst b/docs/zh_cn/api/metrics.rst
index a7a275d7..96ac5ddb 100644
--- a/docs/zh_cn/api/metrics.rst
+++ b/docs/zh_cn/api/metrics.rst
@@ -31,6 +31,7 @@ Metrics
    OIDMeanAP
    F1Metric
    HmeanIoU
+   InstanceSeg
    EndPointError
    PCKAccuracy
    MpiiPCKAccuracy

From 9c6fe5aec3b4c24533145688c53dfd7a054e3b3c Mon Sep 17 00:00:00 2001
From: Pzzzzz <31173671+Pzzzzz5142@users.noreply.github.com>
Date: Sun, 8 Jan 2023 17:18:45 +0800
Subject: [PATCH 14/19] Update metrics.rst

---
 docs/en/api/metrics.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/en/api/metrics.rst b/docs/en/api/metrics.rst
index a7a275d7..96ac5ddb 100644
--- a/docs/en/api/metrics.rst
+++ b/docs/en/api/metrics.rst
@@ -31,6 +31,7 @@ Metrics
    OIDMeanAP
    F1Metric
    HmeanIoU
+   InstanceSeg
    EndPointError
    PCKAccuracy
    MpiiPCKAccuracy

From 9a775e5664d1e3f64d7e6a192083d21533af6e7a Mon Sep 17 00:00:00 2001
From: Pzzzzz <31173671+Pzzzzz5142@users.noreply.github.com>
Date: Wed, 18 Jan 2023 16:14:43 +0800
Subject: [PATCH 15/19] Update mmeval/metrics/instance_seg.py

Co-authored-by: yancong <32220263+ice-tong@users.noreply.github.com>
---
 mmeval/metrics/instance_seg.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/mmeval/metrics/instance_seg.py b/mmeval/metrics/instance_seg.py
index 5451f198..b01d74db 100644
--- a/mmeval/metrics/instance_seg.py
+++ b/mmeval/metrics/instance_seg.py
@@ -229,9 +229,7 @@ def rename_gt(self, gt_semantic_masks, gt_instance_masks, valid_class_ids):
                 semantic_unique = np.unique(semantic_instance)
                 assert len(semantic_unique) == 1
                 if semantic_unique[0] < len(valid_class_ids):
-                    instance_mask[
-                        instance_mask ==
-                        i] = 1000 * valid_class_ids[semantic_unique[0]] + i
+                    instance_mask[instance_mask==i] = 1000 * valid_class_ids[semantic_unique[0]] + i  #noqa: E501
             renamed_instance_masks.append(instance_mask)
         return renamed_instance_masks
 

From 8e7c609843e87ad7f8498f1eecbe36658cd6852a Mon Sep 17 00:00:00 2001
From: Pzzzzz <31173671+Pzzzzz5142@users.noreply.github.com>
Date: Wed, 18 Jan 2023 16:15:13 +0800
Subject: [PATCH 16/19] Update mmeval/metrics/instance_seg.py

Co-authored-by: yancong <32220263+ice-tong@users.noreply.github.com>
---
 mmeval/metrics/instance_seg.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/mmeval/metrics/instance_seg.py b/mmeval/metrics/instance_seg.py
index b01d74db..2e0d83d1 100644
--- a/mmeval/metrics/instance_seg.py
+++ b/mmeval/metrics/instance_seg.py
@@ -134,16 +134,16 @@ def add(self, predictions: Sequence[Dict], groundtruths: Sequence[Dict]) -> None
             predictions (Sequence[Dict]): A sequence of dict. Each dict
                 representing a detection result, with the following keys:
 
-                - pts_instance_mask(np.ndarray): Predicted instance masks.
-                - instance_labels(np.ndarray): Predicted instance labels.
-                - instance_scores(np.ndarray): Predicted instance scores.
+                - pts_instance_mask (np.ndarray): Predicted instance masks.
+                - instance_labels (np.ndarray): Predicted instance labels.
+                - instance_scores (np.ndarray): Predicted instance scores.
 
             groundtruths (Sequence[Dict]): A sequence of dict. Each dict
                 represents a groundtruths for an image, with the following
                 keys:
 
-                - pts_instance_mask(np.ndarray): Ground truth instance masks.
-                - pts_semantic_mask(np.ndarray): Ground truth semantic masks.
+                - pts_instance_mask (np.ndarray): Ground truth instance masks.
+                - pts_semantic_mask (np.ndarray): Ground truth semantic masks.
         """
         for prediction, groundtruth in zip(predictions, groundtruths):
             self._results.append((deepcopy(prediction), deepcopy(groundtruth)))

From c74b444a4340be854869523db62043db0e8756b3 Mon Sep 17 00:00:00 2001
From: Pzzzzz <31173671+Pzzzzz5142@users.noreply.github.com>
Date: Wed, 18 Jan 2023 16:17:53 +0800
Subject: [PATCH 17/19] Update mmeval/metrics/instance_seg.py

Co-authored-by: yancong <32220263+ice-tong@users.noreply.github.com>
---
 mmeval/metrics/instance_seg.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/mmeval/metrics/instance_seg.py b/mmeval/metrics/instance_seg.py
index 2e0d83d1..482d243f 100644
--- a/mmeval/metrics/instance_seg.py
+++ b/mmeval/metrics/instance_seg.py
@@ -17,6 +17,7 @@ class InstanceSeg(BaseMetric):
             an alternative to dataset_meta.
         valid_class_ids (List[int], optional): Provide dataset valid class ids
             information as an alternative to dataset_meta.
+        **kwargs: Keyword parameters passed to :class:`BaseMetric`.
 
     Example:
         >>> import numpy as np

From 3444a3ee90510e1ac9b2cd1b7fd692876dd92386 Mon Sep 17 00:00:00 2001
From: Pzzzzz <hello-cd.plus@hotmail.com>
Date: Wed, 18 Jan 2023 18:02:26 +0800
Subject: [PATCH 18/19] update

---
 mmeval/metrics/instance_seg.py          | 182 ++++++++++++------------
 tests/test_metrics/test_instance_seg.py |  14 --
 2 files changed, 88 insertions(+), 108 deletions(-)

diff --git a/mmeval/metrics/instance_seg.py b/mmeval/metrics/instance_seg.py
index 2e0d83d1..62fe9d6a 100644
--- a/mmeval/metrics/instance_seg.py
+++ b/mmeval/metrics/instance_seg.py
@@ -1,6 +1,5 @@
 # Copyright (c) OpenMMLab. All rights reserved.
 import numpy as np
-import warnings
 from copy import deepcopy
 from typing import Dict, List, Optional, Sequence
 
@@ -11,6 +10,9 @@
 class InstanceSeg(BaseMetric):
     """3D instance segmentation evaluation metric.
 
+    This metric is for ScanNet 3D instance segmentation tasks. For more info
+    about ScanNet, please read [here](https://github.com/ScanNet/ScanNet).
+
     Args:
         dataset_meta (dict, optional): Provide dataset meta information.
         classes (List[str], optional): Provide dataset classes information as
@@ -21,19 +23,15 @@ class InstanceSeg(BaseMetric):
     Example:
         >>> import numpy as np
         >>> from mmeval import InstanceSegMetric
-        >>> seg_valid_class_ids = (3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 14, 16, 24,
-        >>>                 28, 33, 34, 36, 39)
-        >>> class_labels = ('cabinet', 'bed', 'chair', 'sofa', 'table', 'door',
-        ...         'window', 'bookshelf', 'picture', 'counter', 'desk',
-        ...         'curtain', 'refrigerator', 'showercurtrain', 'toilet',
-        ...         'sink', 'bathtub', 'garbagebin')
+        >>> seg_valid_class_ids = (3, 4, 5)
+        >>> class_labels = ('cabinet', 'bed', 'chair')
         >>> dataset_meta = dict(
         ...     seg_valid_class_ids=seg_valid_class_ids, classes=class_labels)
         >>>
         >>> def _demo_mm_model_output(self):
         ...     n_points_list = [3300, 3000]
-        ...     gt_labels_list = [[0, 0, 0, 0, 0, 0, 14, 14, 2, 1],
-        ...                       [13, 13, 2, 1, 3, 3, 0, 0, 0]]
+        ...     gt_labels_list = [[0, 0, 0, 0, 0, 0, 0, 0, 2, 1],
+        ...                       [2, 2, 2, 1, 0, 0, 0, 0, 0]]
         ...
         ...     predictions = []
         ...     groundtruths = []
@@ -78,9 +76,9 @@ class InstanceSeg(BaseMetric):
         >>> res = instance_seg_metric(predictions, groundtruths)
         >>> res
         {
-        'all_ap': 1.0,
-        'all_ap_50%': 1.0,
-        'all_ap_25%': 1.0,
+        'all_ap': 0.8333333333333334,
+        'all_ap_50%': 0.8333333333333334,
+        'all_ap_25%': 0.8333333333333334,
         'classes': {
             'cabinet': {
                 'ap': 1.0,
@@ -88,18 +86,17 @@ class InstanceSeg(BaseMetric):
                 'ap25%': 1.0
             },
             'bed': {
-                'ap': 1.0,
-                'ap50%': 1.0,
-                'ap25%': 1.0
+                'ap': 0.5,
+                'ap50%': 0.5,
+                'ap25%': 0.5
             },
             'chair': {
                 'ap': 1.0,
                 'ap50%': 1.0,
                 'ap25%': 1.0
-            },
-            ...
+            }
+        }
         }
-    }
     """
 
     def __init__(self,
@@ -107,21 +104,56 @@ def __init__(self,
                  valid_class_ids: Optional[List[int]] = None,
                  **kwargs):
         super().__init__(**kwargs)
-        assert (self.dataset_meta
-                is not None) or (classes is not None
-                                 and valid_class_ids is not None)
-        if self.dataset_meta is not None:
-            if classes is not None or valid_class_ids is not None:
-                warnings.warn('`classes` and `valid_class_ids` are ignored.')
-            self.classes = self.dataset_meta['classes']
-            self.valid_class_ids = self.dataset_meta['seg_valid_class_ids']
+        self._valid_class_ids = valid_class_ids
+        self._classes = classes
+
+    @property
+    def classes(self):
+        """Returns classes.
+
+        The classes should be set during initialization, otherwise it will
+        be obtained from the 'classes' field in ``self.dataset_meta``.
+
+        Raises:
+            RuntimeError: If the classes is not set.
+
+        Returns:
+            List[str]: The classes.
+        """
+        if self._classes is not None:
+            return self._classes
+
+        if self.dataset_meta and 'classes' in self.dataset_meta:
+            self._classes = self.dataset_meta['classes']
         else:
-            if not isinstance(classes, list):
-                raise TypeError('`classes` should be List[str]')
-            if not isinstance(valid_class_ids, list):
-                raise TypeError('`valid_class_ids` should be List[int]')
-            self.classes = classes
-            self.valid_class_ids = valid_class_ids
+            raise RuntimeError('The `classes` is required, and not found in '
+                               f'dataset_meta: {self.dataset_meta}')
+        return self._classes
+
+    @property
+    def valid_class_ids(self):
+        """Returns valid class ids.
+
+        The valid class ids should be set during initialization, otherwise
+        it will be obtained from the 'seg_valid_class_ids' field in
+        ``self.dataset_meta``.
+
+        Raises:
+            RuntimeError: If valid class ids is not set.
+
+        Returns:
+            List[str]: The valid class ids.
+        """
+        if self._classes is not None:
+            return self._valid_class_ids
+
+        if self.dataset_meta and 'seg_valid_class_ids' in self.dataset_meta:
+            self._valid_class_ids = self.dataset_meta['seg_valid_class_ids']
+        else:
+            raise RuntimeError(
+                'The `seg_valid_class_ids` is required, and not found in '
+                f'dataset_meta: {self.dataset_meta}')
+        return self._valid_class_ids
 
     def add(self, predictions: Sequence[Dict], groundtruths: Sequence[Dict]) -> None:  # type: ignore # yapf: disable # noqa: E501
         """Process one batch of data samples and predictions.
@@ -171,16 +203,27 @@ def compute_metric(self, results: List[List[Dict]]) -> Dict[str, float]:
             pred_instance_labels.append(result_pred['instance_labels'])
             pred_instance_scores.append(result_pred['instance_scores'])
 
-        ret_dict = self.instance_seg_eval(
-            gt_semantic_masks,
-            gt_instance_masks,
-            pred_instance_masks,
-            pred_instance_labels,
-            pred_instance_scores,
+        assert len(self.valid_class_ids) == len(self.classes)
+        id_to_label = {
+            self.valid_class_ids[i]: self.classes[i]
+            for i in range(len(self.valid_class_ids))
+        }
+        preds = self.aggregate_predictions(
+            masks=pred_instance_masks,
+            labels=pred_instance_labels,
+            scores=pred_instance_scores,
+            valid_class_ids=self.valid_class_ids)
+        gts = self.rename_gt(gt_semantic_masks, gt_instance_masks,
+                             self.valid_class_ids)
+        metrics = scannet_eval(
+            preds=preds,
+            gts=gts,
+            options=None,
             valid_class_ids=self.valid_class_ids,
-            class_labels=self.classes)
+            class_labels=self.classes,
+            id_to_label=id_to_label)
 
-        return ret_dict
+        return metrics
 
     def aggregate_predictions(self, masks, labels, scores, valid_class_ids):
         """Maps predictions to ScanNet evaluator format.
@@ -223,64 +266,15 @@ def rename_gt(self, gt_semantic_masks, gt_instance_masks, valid_class_ids):
         for semantic_mask, instance_mask in zip(gt_semantic_masks,
                                                 gt_instance_masks):
             unique = np.unique(instance_mask)
-            assert len(unique) < 1000
+            assert len(
+                unique
+            ) < 1000, 'The nums of label in gt should not be greater than 1000'
             for i in unique:
                 semantic_instance = semantic_mask[instance_mask == i]
                 semantic_unique = np.unique(semantic_instance)
                 assert len(semantic_unique) == 1
                 if semantic_unique[0] < len(valid_class_ids):
-                    instance_mask[instance_mask==i] = 1000 * valid_class_ids[semantic_unique[0]] + i  #noqa: E501
+                    instance_mask[instance_mask == i] = 1000 * valid_class_ids[
+                        semantic_unique[0]] + i  # noqa: E501
             renamed_instance_masks.append(instance_mask)
         return renamed_instance_masks
-
-    def instance_seg_eval(self,
-                          gt_semantic_masks,
-                          gt_instance_masks,
-                          pred_instance_masks,
-                          pred_instance_labels,
-                          pred_instance_scores,
-                          valid_class_ids,
-                          class_labels,
-                          options=None):
-        """Instance Segmentation Evaluation.
-
-        Evaluate the result of the instance segmentation.
-
-        Args:
-            gt_semantic_masks (list[np.ndarray]): Ground truth semantic masks.
-            gt_instance_masks (list[np.ndarray]): Ground truth instance masks.
-            pred_instance_masks (list[np.ndarray]): Predicted instance masks.
-            pred_instance_labels (list[np.ndarray]): Predicted instance labels.
-            pred_instance_scores (list[np.ndarray]): Predicted instance scores.
-            valid_class_ids (tuple[int]): Ids of valid categories.
-            class_labels (tuple[str]): Names of valid categories.
-            options (dict, optional): Additional options. Keys may contain:
-                `overlaps`, `min_region_sizes`, `distance_threshes`,
-                `distance_confs`. Default: None.
-            logger (logging.Logger | str, optional): The way to print the
-                mAP summary. See `mmdet.utils.print_log()` for details.
-                Default: None.
-
-        Returns:
-            dict[str, float]: Dict of results.
-        """
-        assert len(valid_class_ids) == len(class_labels)
-        id_to_label = {
-            valid_class_ids[i]: class_labels[i]
-            for i in range(len(valid_class_ids))
-        }
-        preds = self.aggregate_predictions(
-            masks=pred_instance_masks,
-            labels=pred_instance_labels,
-            scores=pred_instance_scores,
-            valid_class_ids=valid_class_ids)
-        gts = self.rename_gt(gt_semantic_masks, gt_instance_masks,
-                             valid_class_ids)
-        metrics = scannet_eval(
-            preds=preds,
-            gts=gts,
-            options=options,
-            valid_class_ids=valid_class_ids,
-            class_labels=class_labels,
-            id_to_label=id_to_label)
-        return metrics
diff --git a/tests/test_metrics/test_instance_seg.py b/tests/test_metrics/test_instance_seg.py
index f66407b9..1d0860f6 100644
--- a/tests/test_metrics/test_instance_seg.py
+++ b/tests/test_metrics/test_instance_seg.py
@@ -1,6 +1,5 @@
 # Copyright (c) OpenMMLab. All rights reserved.
 import numpy as np
-import pytest
 import torch
 
 from mmeval.metrics import InstanceSeg
@@ -42,19 +41,6 @@ def _demo_mm_model_output():
     return predictions, groundtruths
 
 
-def test_metric_invalid_usage():
-    with pytest.raises(AssertionError):
-        InstanceSeg()
-
-    with pytest.raises(AssertionError):
-        InstanceSeg(
-            valid_class_ids=(3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 14, 16, 24, 28,
-                             33, 34, 36, 39))
-
-    with pytest.raises(TypeError):
-        InstanceSeg(classes='a', valid_class_ids='b')
-
-
 def test_evaluate():
     predictions, groundtruths = _demo_mm_model_output()
     predictions = [{k: v.clone().numpy()

From 3378a498abea9c3d16a7bf286ae3134f5f17d1a1 Mon Sep 17 00:00:00 2001
From: Pzzzzz <hello-cd.plus@hotmail.com>
Date: Wed, 15 Feb 2023 21:48:41 +0800
Subject: [PATCH 19/19] update

---
 mmeval/metrics/instance_seg.py          | 173 ++++++------------------
 tests/test_metrics/test_instance_seg.py | 110 ++++++++-------
 2 files changed, 103 insertions(+), 180 deletions(-)

diff --git a/mmeval/metrics/instance_seg.py b/mmeval/metrics/instance_seg.py
index bd393d68..4e591601 100644
--- a/mmeval/metrics/instance_seg.py
+++ b/mmeval/metrics/instance_seg.py
@@ -1,5 +1,4 @@
 # Copyright (c) OpenMMLab. All rights reserved.
-import numpy as np
 from copy import deepcopy
 from typing import Dict, List, Optional, Sequence
 
@@ -29,57 +28,38 @@ class InstanceSeg(BaseMetric):
         >>> dataset_meta = dict(
         ...     seg_valid_class_ids=seg_valid_class_ids, classes=class_labels)
         >>>
-        >>> def _demo_mm_model_output(self):
-        ...     n_points_list = [3300, 3000]
-        ...     gt_labels_list = [[0, 0, 0, 0, 0, 0, 0, 0, 2, 1],
-        ...                       [2, 2, 2, 1, 0, 0, 0, 0, 0]]
-        ...
-        ...     predictions = []
-        ...     groundtruths = []
-        ...
-        ...     for n_points, gt_labels in zip(n_points_list, gt_labels_list):
-        ...         gt_instance_mask = np.ones(n_points, dtype=int) * -1
-        ...         gt_semantic_mask = np.ones(n_points, dtype=int) * -1
-        ...         for i, gt_label in enumerate(gt_labels):
-        ...             begin = i * 300
-        ...             end = begin + 300
-        ...             gt_instance_mask[begin:end] = i
-        ...             gt_semantic_mask[begin:end] = gt_label
-        ...
-        ...         ann_info_data = dict()
-        ...         ann_info_data['pts_instance_mask'] = torch.tensor(
-        ...             gt_instance_mask)
-        ...         ann_info_data['pts_semantic_mask'] = torch.tensor(
-        ...             gt_semantic_mask)
-        ...
-        ...         results_dict = dict()
-        ...         pred_instance_mask = np.ones(n_points, dtype=int) * -1
-        ...         labels = []
-        ...         scores = []
-        ...         for i, gt_label in enumerate(gt_labels):
-        ...             begin = i * 300
-        ...             end = begin + 300
-        ...             pred_instance_mask[begin:end] = i
-        ...             labels.append(gt_label)
-        ...             scores.append(.99)
-        ...
-        ...         results_dict['pts_instance_mask'] = torch.tensor(
-        ...             pred_instance_mask)
-        ...         results_dict['instance_labels'] = torch.tensor(labels)
-        ...         results_dict['instance_scores'] = torch.tensor(scores)
-        ...
-        ...         predictions.append(results_dict)
-        ...         groundtruths.append(ann_info_data)
-        ...
-        ...     return predictions, groundtruths
+        >>> def _demo_mm_model_output():
+        >>>     n_points_list = [3300, 3000]
+        >>>     gt_labels_list = [[0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 2, 0],
+        >>>                       [1, 1, 2, 1, 2, 2, 0, 0, 0, 0, 1]]
+        >>>     predictions = []
+        >>>     groundtruths = []
+        >>>
+        >>>     for idx, points_num in enumerate(n_points_list):
+        >>>         points = np.ones(points_num) * -1
+        >>>         gt = np.ones(points_num)
+        >>>         info = {}
+        >>>         for ii, i in enumerate(gt_labels_list[idx]):
+        >>>             i = seg_valid_class_ids[i]
+        >>>             points[ii * 300:(ii + 1) * 300] = ii
+        >>>             gt[ii * 300:(ii + 1) * 300] = i * 1000 + ii
+        >>>             info[f"{idx}_{ii}"] = {
+        >>>                 'mask': (points == ii),
+        >>>                 'label_id': i,
+        >>>                 'conf': 0.99
+        >>>             }
+        >>>         predictions.append(info)
+        >>>         groundtruths.append(gt)
+        >>>
+        >>>     return predictions, groundtruths
         >>>
         >>> instance_seg_metric = InstanceSegMetric(dataset_meta=dataset_meta)
         >>> res = instance_seg_metric(predictions, groundtruths)
         >>> res
         {
-        'all_ap': 0.8333333333333334,
-        'all_ap_50%': 0.8333333333333334,
-        'all_ap_25%': 0.8333333333333334,
+        'all_ap': 1.0,
+        'all_ap_50%': 1.0,
+        'all_ap_25%': 1.0,
         'classes': {
             'cabinet': {
                 'ap': 1.0,
@@ -145,7 +125,7 @@ def valid_class_ids(self):
         Returns:
             List[str]: The valid class ids.
         """
-        if self._classes is not None:
+        if self._valid_class_ids is not None:
             return self._valid_class_ids
 
         if self.dataset_meta and 'seg_valid_class_ids' in self.dataset_meta:
@@ -165,18 +145,16 @@ def add(self, predictions: Sequence[Dict], groundtruths: Sequence[Dict]) -> None
 
         Args:
             predictions (Sequence[Dict]): A sequence of dict. Each dict
-                representing a detection result, with the following keys:
+                representing a detection result. The dict has multiple keys,
+                each key represents the name of the instance, and the value
+                is also a dict, with following keys:
 
-                - pts_instance_mask (np.ndarray): Predicted instance masks.
-                - instance_labels (np.ndarray): Predicted instance labels.
-                - instance_scores (np.ndarray): Predicted instance scores.
+                - mask (array): Predicted instance masks.
+                - label_id (int): Predicted instance labels.
+                - conf (float): Predicted instance scores.
 
-            groundtruths (Sequence[Dict]): A sequence of dict. Each dict
-                represents a groundtruths for an image, with the following
-                keys:
-
-                - pts_instance_mask (np.ndarray): Ground truth instance masks.
-                - pts_semantic_mask (np.ndarray): Ground truth semantic masks.
+            groundtruths (Sequence[array]): A sequence of array. Each array
+                represents a groundtruths for an image.
         """
         for prediction, groundtruth in zip(predictions, groundtruths):
             self._results.append((deepcopy(prediction), deepcopy(groundtruth)))
@@ -191,31 +169,18 @@ def compute_metric(self, results: List[List[Dict]]) -> Dict[str, float]:
             Dict[str, float]: The computed metrics. The keys are the names of
             the metrics, and the values are corresponding results.
         """
-        gt_semantic_masks = []
-        gt_instance_masks = []
-        pred_instance_masks = []
-        pred_instance_labels = []
-        pred_instance_scores = []
-
-        for result_pred, result_gt in results:
-            gt_semantic_masks.append(result_gt['pts_semantic_mask'])
-            gt_instance_masks.append(result_gt['pts_instance_mask'])
-            pred_instance_masks.append(result_pred['pts_instance_mask'])
-            pred_instance_labels.append(result_pred['instance_labels'])
-            pred_instance_scores.append(result_pred['instance_scores'])
+        preds = []
+        gts = []
+        for pred, gt in results:
+            preds.append(pred)
+            gts.append(gt)
 
         assert len(self.valid_class_ids) == len(self.classes)
         id_to_label = {
             self.valid_class_ids[i]: self.classes[i]
             for i in range(len(self.valid_class_ids))
         }
-        preds = self.aggregate_predictions(
-            masks=pred_instance_masks,
-            labels=pred_instance_labels,
-            scores=pred_instance_scores,
-            valid_class_ids=self.valid_class_ids)
-        gts = self.rename_gt(gt_semantic_masks, gt_instance_masks,
-                             self.valid_class_ids)
+
         metrics = scannet_eval(
             preds=preds,
             gts=gts,
@@ -225,57 +190,3 @@ def compute_metric(self, results: List[List[Dict]]) -> Dict[str, float]:
             id_to_label=id_to_label)
 
         return metrics
-
-    def aggregate_predictions(self, masks, labels, scores, valid_class_ids):
-        """Maps predictions to ScanNet evaluator format.
-
-        Args:
-            masks (list[np.ndarray]): Per scene predicted instance masks.
-            labels (list[np.ndarray]): Per scene predicted instance labels.
-            scores (list[np.ndarray]): Per scene predicted instance scores.
-            valid_class_ids (tuple[int]): Ids of valid categories.
-
-        Returns:
-            list[dict]: Per scene aggregated predictions.
-        """
-        infos = []
-        for id, (mask, label, score) in enumerate(zip(masks, labels, scores)):
-            info = dict()
-            n_instances = mask.max() + 1
-            for i in range(n_instances):
-                file_name = f'{id}_{i}'
-                info[file_name] = dict()
-                info[file_name]['mask'] = (mask == i).astype(int)
-                info[file_name]['label_id'] = valid_class_ids[label[i]]
-                info[file_name]['conf'] = score[i]
-            infos.append(info)
-        return infos
-
-    def rename_gt(self, gt_semantic_masks, gt_instance_masks, valid_class_ids):
-        """Maps gt instance and semantic masks to instance masks for ScanNet
-        evaluator.
-
-        Args:
-            gt_semantic_masks (list[np.ndarray]): Per scene gt semantic masks.
-            gt_instance_masks (list[np.ndarray]): Per scene gt instance masks.
-            valid_class_ids (tuple[int]): Ids of valid categories.
-
-        Returns:
-            list[np.array]: Per scene instance masks.
-        """
-        renamed_instance_masks = []
-        for semantic_mask, instance_mask in zip(gt_semantic_masks,
-                                                gt_instance_masks):
-            unique = np.unique(instance_mask)
-            assert len(
-                unique
-            ) < 1000, 'The nums of label in gt should not be greater than 1000'
-            for i in unique:
-                semantic_instance = semantic_mask[instance_mask == i]
-                semantic_unique = np.unique(semantic_instance)
-                assert len(semantic_unique) == 1
-                if semantic_unique[0] < len(valid_class_ids):
-                    instance_mask[instance_mask == i] = 1000 * valid_class_ids[
-                        semantic_unique[0]] + i  # noqa: E501
-            renamed_instance_masks.append(instance_mask)
-        return renamed_instance_masks
diff --git a/tests/test_metrics/test_instance_seg.py b/tests/test_metrics/test_instance_seg.py
index 1d0860f6..3311cfdb 100644
--- a/tests/test_metrics/test_instance_seg.py
+++ b/tests/test_metrics/test_instance_seg.py
@@ -1,60 +1,74 @@
 # Copyright (c) OpenMMLab. All rights reserved.
 import numpy as np
-import torch
 
 from mmeval.metrics import InstanceSeg
 
+seg_valid_class_ids = (3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 14, 16, 24, 28, 33, 34,
+                       36, 39)
+class_labels = ('cabinet', 'bed', 'chair', 'sofa', 'table', 'door', 'window',
+                'bookshelf', 'picture', 'counter', 'desk', 'curtain',
+                'refrigerator', 'showercurtrain', 'toilet', 'sink', 'bathtub',
+                'garbagebin')
+dataset_meta = dict(
+    seg_valid_class_ids=seg_valid_class_ids, classes=class_labels)
+
 
 def _demo_mm_model_output():
     """Create a superset of inputs needed to run test or train batches."""
     n_points_list = [3300, 3000]
-    gt_labels_list = [[0, 0, 0, 0, 0, 0, 14, 14, 2, 1],
-                      [13, 13, 2, 1, 3, 3, 0, 0, 0]]
+    gt_labels_list = [[0, 0, 0, 0, 0, 0, 14, 14, 2, 2, 2],
+                      [13, 13, 2, 1, 3, 3, 0, 0, 0, 0]]
+    predictions = []
+    groundtruths = []
+
+    for idx, points_num in enumerate(n_points_list):
+        points = np.ones(points_num) * -1
+        gt = np.ones(points_num)
+        info = {}
+        for ii, i in enumerate(gt_labels_list[idx]):
+            i = seg_valid_class_ids[i]
+            points[ii * 300:(ii + 1) * 300] = ii
+            gt[ii * 300:(ii + 1) * 300] = i * 1000 + ii
+            info[f'{idx}_{ii}'] = {
+                'mask': (points == ii),
+                'label_id': i,
+                'conf': 0.99
+            }
+        predictions.append(info)
+        groundtruths.append(gt)
+
+    return predictions, groundtruths
+
+
+def _demo_mm_model_wrong_output():
+    """Create a superset of inputs needed to run test or train batches."""
+    n_points_list = [3300, 3000]
+    gt_labels_list = [[0, 0, 0, 0, 0, 0, 14, 14, 2, 2, 2],
+                      [13, 13, 2, 1, 3, 3, 0, 0, 0, 0]]
     predictions = []
     groundtruths = []
-    for n_points, gt_labels in zip(n_points_list, gt_labels_list):
-        gt_instance_mask = np.ones(n_points, dtype=int) * -1
-        gt_semantic_mask = np.ones(n_points, dtype=int) * -1
-        for i, gt_label in enumerate(gt_labels):
-            begin = i * 300
-            end = begin + 300
-            gt_instance_mask[begin:end] = i
-            gt_semantic_mask[begin:end] = gt_label
-        ann_info_data = dict()
-        ann_info_data['pts_instance_mask'] = torch.tensor(gt_instance_mask)
-        ann_info_data['pts_semantic_mask'] = torch.tensor(gt_semantic_mask)
-        results_dict = dict()
-        pred_instance_mask = np.ones(n_points, dtype=int) * -1
-        labels = []
-        scores = []
-        for i, gt_label in enumerate(gt_labels):
-            begin = i * 300
-            end = begin + 300
-            pred_instance_mask[begin:end] = i
-            labels.append(gt_label)
-            scores.append(.99)
-        results_dict['pts_instance_mask'] = torch.tensor(pred_instance_mask)
-        results_dict['instance_labels'] = torch.tensor(labels)
-        results_dict['instance_scores'] = torch.tensor(scores)
-        predictions.append(results_dict)
-        groundtruths.append(ann_info_data)
+
+    for idx, points_num in enumerate(n_points_list):
+        points = np.ones(points_num) * -1
+        gt = np.ones(points_num)
+        info = {}
+        for ii, i in enumerate(gt_labels_list[idx]):
+            i = seg_valid_class_ids[i]
+            points[ii * 300:(ii + 1) * 300] = i
+            gt[ii * 300:(ii + 1) * 300] = i * 1000 + ii
+            info[f'{idx}_{ii}'] = {
+                'mask': (points == i),
+                'label_id': i,
+                'conf': 0.99
+            }
+        predictions.append(info)
+        groundtruths.append(gt)
+
     return predictions, groundtruths
 
 
 def test_evaluate():
     predictions, groundtruths = _demo_mm_model_output()
-    predictions = [{k: v.clone().numpy()
-                    for k, v in i.items()} for i in predictions]
-    groundtruths = [{k: v.clone().numpy()
-                     for k, v in i.items()} for i in groundtruths]
-    seg_valid_class_ids = (3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 14, 16, 24, 28, 33,
-                           34, 36, 39)
-    class_labels = ('cabinet', 'bed', 'chair', 'sofa', 'table', 'door',
-                    'window', 'bookshelf', 'picture', 'counter', 'desk',
-                    'curtain', 'refrigerator', 'showercurtrain', 'toilet',
-                    'sink', 'bathtub', 'garbagebin')
-    dataset_meta = dict(
-        seg_valid_class_ids=seg_valid_class_ids, classes=class_labels)
     instance_seg_metric = InstanceSeg(dataset_meta=dataset_meta)
     res = instance_seg_metric(predictions, groundtruths)
     assert isinstance(res, dict)
@@ -65,13 +79,11 @@ def test_evaluate():
         assert metrics['ap'] == 1.0
         assert metrics['ap50%'] == 1.0
         assert metrics['ap25%'] == 1.0
-    predictions[1]['pts_instance_mask'][2240:2700] = -1
-    predictions[0]['pts_instance_mask'][2700:3000] = 8
-    predictions[0]['instance_labels'][9] = 2
+    predictions, groundtruths = _demo_mm_model_wrong_output()
     res = instance_seg_metric(predictions, groundtruths)
-    assert abs(res['classes']['cabinet']['ap50%'] - 0.72916) < 0.01
-    assert abs(res['classes']['cabinet']['ap25%'] - 0.88888) < 0.01
-    assert abs(res['classes']['bed']['ap50%'] - 0.5) < 0.01
-    assert abs(res['classes']['bed']['ap25%'] - 0.5) < 0.01
+    assert abs(res['classes']['cabinet']['ap50%'] - 0.12) < 0.01
+    assert abs(res['classes']['cabinet']['ap25%'] - 0.4125) < 0.01
+    assert abs(res['classes']['bed']['ap50%'] - 1) < 0.01
+    assert abs(res['classes']['bed']['ap25%'] - 1) < 0.01
     assert abs(res['classes']['chair']['ap50%'] - 0.375) < 0.01
-    assert abs(res['classes']['chair']['ap25%'] - 1.0) < 0.01
+    assert abs(res['classes']['chair']['ap25%'] - 0.785714) < 0.01