From 3f8f5045d3a5b8b83ad907564c8b441edcbb6f7d Mon Sep 17 00:00:00 2001 From: Helicopt Date: Tue, 16 Apr 2019 17:52:44 +0800 Subject: [PATCH] Faster merging procedure, more detailed metrics analysis and more compatible for MOT16/17 (#38) * more like official ver * renew readme * make same result as MATLAB * rm useless * faster distance matrix & faster IDF1 computing * ... * fix short result * fix situation of nan * fix nan * version bump * fix realease readme * add idt ida * fix IDt * fix typo * fix hypHis * fix gt number * speed up merge overall * add skip func * add special iou requirement * fix old tests * add tests for id s t a m * rm false assertion * change interface back to origin design * fix row names --- Readme.md | 8 +- motmetrics/apps/evaluateTracking.py | 164 ++++++++++++++ motmetrics/distances.py | 37 +++- motmetrics/io.py | 7 +- motmetrics/metrics.py | 317 ++++++++++++++++++++++++---- motmetrics/mot.py | 175 +++++++++------ motmetrics/preprocess.py | 65 ++++++ motmetrics/tests/test_metrics.py | 38 +++- motmetrics/tests/test_mot.py | 22 +- motmetrics/utils.py | 88 +++++++- seqmaps/example.txt | 3 + test.sh.example | 1 + 12 files changed, 804 insertions(+), 121 deletions(-) create mode 100644 motmetrics/apps/evaluateTracking.py create mode 100644 motmetrics/preprocess.py create mode 100644 seqmaps/example.txt create mode 100644 test.sh.example diff --git a/Readme.md b/Readme.md index 18515bdd..24c8fbaf 100644 --- a/Readme.md +++ b/Readme.md @@ -10,7 +10,8 @@ While benchmarking single object trackers is rather straightforward, measuring t
-![](motmetrics/etc/mot.png)
+![](./motmetrics/etc/mot.png)
+ *Pictures courtesy of Bernardin, Keni, and Rainer Stiefelhagen [[1]](#References)*
@@ -103,6 +104,10 @@ You can compare tracker results to ground truth in MOTChallenge format by ``` python -m motmetrics.apps.eval_motchallenge --help ``` +For MOT16/17, you can run +``` +python -m motmetrics.apps.evaluateTracking --help +``` ### Installation @@ -450,6 +455,7 @@ docker run desired-image-name MIT License Copyright (c) 2017 Christoph Heindl +Copyright (c) 2018 Toka Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/motmetrics/apps/evaluateTracking.py b/motmetrics/apps/evaluateTracking.py new file mode 100644 index 00000000..4ce527aa --- /dev/null +++ b/motmetrics/apps/evaluateTracking.py @@ -0,0 +1,164 @@ +"""py-motmetrics - metrics for multiple object tracker (MOT) benchmarking with RESULT PREPROCESS. + +TOKA, 2018 +ORIGIN: https://github.com/cheind/py-motmetrics +EXTENDED: +""" + +import argparse +import glob +import os +import logging +import motmetrics as mm +import pandas as pd +from collections import OrderedDict +from pathlib import Path +import time +from tempfile import NamedTemporaryFile + +def parse_args(): + parser = argparse.ArgumentParser(description=""" +Compute metrics for trackers using MOTChallenge ground-truth data with data preprocess. + +Files +----- +All file content, ground truth and test files, have to comply with the +format described in + +Milan, Anton, et al. +"Mot16: A benchmark for multi-object tracking." +arXiv preprint arXiv:1603.00831 (2016). +https://motchallenge.net/ + +Structure +--------- + +Layout for ground truth data + //gt/gt.txt + //gt/gt.txt + ... + +Layout for test data + /.txt + /.txt + ... + +Seqmap for test data + [name] + + + ... + +Sequences of ground truth and test will be matched according to the `` +string in the seqmap.""", formatter_class=argparse.RawTextHelpFormatter) + + parser.add_argument('groundtruths', type=str, help='Directory containing ground truth files.') + parser.add_argument('tests', type=str, help='Directory containing tracker result files') + parser.add_argument('seqmap', type=str, help='Text file containing all sequences name') + parser.add_argument('--log', type=str, help='a place to record result and outputfile of mistakes', default='') + parser.add_argument('--loglevel', type=str, help='Log level', default='info') + parser.add_argument('--fmt', type=str, help='Data format', default='mot15-2D') + parser.add_argument('--solver', type=str, help='LAP solver to use') + parser.add_argument('--skip', type=int, default=0, help='skip frames n means choosing one frame for every (n+1) frames') + parser.add_argument('--iou', type=float, default=0.5, help='special IoU threshold requirement for small targets') + return parser.parse_args() + +def compare_dataframes(gts, ts, vsflag = '', iou = 0.5): + accs = [] + anas = [] + names = [] + for k, tsacc in ts.items(): + if k in gts: + logging.info('Evaluating {}...'.format(k)) + if vsflag!='': + fd = open(vsflag+'/'+k+'.log','w') + else: + fd = '' + acc, ana = mm.utils.CLEAR_MOT_M(gts[k][0], tsacc, gts[k][1], 'iou', distth=iou, vflag=fd) + if fd!='': + fd.close() + accs.append(acc) + anas.append(ana) + names.append(k) + else: + logging.warning('No ground truth for {}, skipping.'.format(k)) + + return accs, anas, names + +def parseSequences(seqmap): + assert os.path.isfile(seqmap), 'Seqmap %s not found.'%seqmap + fd = open(seqmap) + res = [] + for row in fd.readlines(): + row = row.strip() + if row=='' or row=='name' or row[0]=='#': continue + res.append(row) + fd.close() + return res + +def generateSkippedGT(gtfile, skip, fmt): + tf = NamedTemporaryFile(delete=False, mode='w') + with open(gtfile) as fd: + lines = fd.readlines() + for line in lines: + arr = line.strip().split(',') + fr = int(arr[0]) + if fr%(skip+1)!=1: + continue + pos = line.find(',') + newline = str(fr//(skip+1)+1) + line[pos:] + tf.write(newline) + tf.close() + tempfile = tf.name + return tempfile + + +if __name__ == '__main__': + + args = parse_args() + + loglevel = getattr(logging, args.loglevel.upper(), None) + if not isinstance(loglevel, int): + raise ValueError('Invalid log level: {} '.format(args.loglevel)) + logging.basicConfig(level=loglevel, format='%(asctime)s %(levelname)s - %(message)s', datefmt='%I:%M:%S') + + if args.solver: + mm.lap.default_solver = args.solver + + seqs = parseSequences(args.seqmap) + gtfiles = [os.path.join(args.groundtruths, i, 'gt/gt.txt') for i in seqs] + tsfiles = [os.path.join(args.tests, '%s.txt'%i) for i in seqs] + + for gtfile in gtfiles: + if not os.path.isfile(gtfile): + logging.error('gt File %s not found.'%gtfile) + exit(1) + for tsfile in tsfiles: + if not os.path.isfile(tsfile): + logging.error('res File %s not found.'%tsfile) + exit(1) + + logging.info('Found {} groundtruths and {} test files.'.format(len(gtfiles), len(tsfiles))) + for seq in seqs: + logging.info('\t%s'%seq) + logging.info('Available LAP solvers {}'.format(mm.lap.available_solvers)) + logging.info('Default LAP solver \'{}\''.format(mm.lap.default_solver)) + logging.info('Loading files.') + + if args.skip>0 and 'mot' in args.fmt: + for i, gtfile in enumerate(gtfiles): + gtfiles[i] = generateSkippedGT(gtfile, args.skip, fmt=args.fmt) + + gt = OrderedDict([(seqs[i], (mm.io.loadtxt(f, fmt=args.fmt), os.path.join(args.groundtruths, seqs[i], 'seqinfo.ini')) ) for i, f in enumerate(gtfiles)]) + ts = OrderedDict([(seqs[i], mm.io.loadtxt(f, fmt=args.fmt)) for i, f in enumerate(tsfiles)]) + + mh = mm.metrics.create() + st = time.time() + accs, analysis, names = compare_dataframes(gt, ts, args.log, 1.-args.iou) + logging.info('adding frames: %.3f seconds.'%(time.time()-st)) + + logging.info('Running metrics') + + summary = mh.compute_many(accs, anas = analysis, names=names, metrics=mm.metrics.motchallenge_metrics, generate_overall=True) + print(mm.io.render_summary(summary, formatters=mh.formatters, namemap=mm.io.motchallenge_metric_names)) + logging.info('Completed') diff --git a/motmetrics/distances.py b/motmetrics/distances.py index 8f5d1590..79d5f3cc 100644 --- a/motmetrics/distances.py +++ b/motmetrics/distances.py @@ -1,7 +1,9 @@ """py-motmetrics - metrics for multiple object tracker (MOT) benchmarking. Christoph Heindl, 2017 +Toka, 2018 https://github.com/cheind/py-motmetrics +Fast implement by TOKA """ import numpy as np @@ -47,6 +49,16 @@ def norm2squared_matrix(objs, hyps, max_d2=float('inf')): C[C > max_d2] = np.nan return C +def boxiou(a, b): + rx1 = max(a[0], b[0]) + rx2 = min(a[0]+a[2], b[0]+b[2]) + ry1 = max(a[1], b[1]) + ry2 = min(a[1]+a[3], b[1]+b[3]) + if ry2>ry1 and rx2>rx1: + i = (ry2-ry1)*(rx2-rx1) + u = a[2]*a[3]+b[2]*b[3]-i + return float(i)/u + else: return 0.0 def iou_matrix(objs, hyps, max_iou=1.): """Computes 'intersection over union (IoU)' distance matrix between object and hypothesis rectangles. @@ -79,9 +91,10 @@ def iou_matrix(objs, hyps, max_iou=1.): Distance matrix containing pairwise distances or np.nan. """ + #import time + #st = time.time() objs = np.atleast_2d(objs).astype(float) hyps = np.atleast_2d(hyps).astype(float) - if objs.size == 0 or hyps.size == 0: return np.empty((0,0)) @@ -95,16 +108,22 @@ def iou_matrix(objs, hyps, max_iou=1.): for o in range(objs.shape[0]): for h in range(hyps.shape[0]): - isect_xy = np.maximum(objs[o, :2], hyps[h, :2]) - isect_wh = np.maximum(np.minimum(br_objs[o], br_hyps[h]) - isect_xy, 0) - isect_a = isect_wh[0]*isect_wh[1] - union_a = objs[o, 2]*objs[o, 3] + hyps[h, 2]*hyps[h, 3] - isect_a - if union_a != 0: - C[o, h] = 1. - isect_a / union_a - else: + #isect_xy = np.maximum(objs[o, :2], hyps[h, :2]) + #isect_wh = np.maximum(np.minimum(br_objs[o], br_hyps[h]) - isect_xy, 0) + #isect_a = isect_wh[0]*isect_wh[1] + #union_a = objs[o, 2]*objs[o, 3] + hyps[h, 2]*hyps[h, 3] - isect_a + #if union_a != 0: + # C[o, h] = 1. - isect_a / union_a + #else: + # C[o, h] = np.nan + iou = boxiou(objs[o], hyps[h]) + if 1 - iou > max_iou: C[o, h] = np.nan + else: + C[o, h] = 1 - iou - C[C > max_iou] = np.nan + #C[C > max_iou] = np.nan + #print('----'*2,'done',time.time()-st) return C diff --git a/motmetrics/io.py b/motmetrics/io.py index 3d1d22db..3427a134 100644 --- a/motmetrics/io.py +++ b/motmetrics/io.py @@ -4,6 +4,8 @@ Christoph Heindl, 2017 https://github.com/cheind/py-motmetrics +Modified by Toka, 2018 +https://github.com/Helicopt/fast-py-MOTMetrics.git """ from enum import Enum @@ -225,6 +227,9 @@ def render_summary(summary, formatters=None, namemap=None, buf=None): 'num_switches' : 'IDs', 'num_fragmentations' : 'FM', 'mota' : 'MOTA', - 'motp' : 'MOTP' + 'motp' : 'MOTP', + 'num_transfer' : 'IDt', + 'num_ascend' : 'IDa', + 'num_migrate' : 'IDm', } """A list mappings for metric names to comply with MOTChallenge.""" diff --git a/motmetrics/metrics.py b/motmetrics/metrics.py index 72a3db2a..2820aabe 100644 --- a/motmetrics/metrics.py +++ b/motmetrics/metrics.py @@ -1,7 +1,9 @@ """py-motmetrics - metrics for multiple object tracker (MOT) benchmarking. Christoph Heindl, 2017 -https://github.com/cheind/py-motmetrics +Toka, 2018 +Origin: https://github.com/cheind/py-motmetrics +Toka make it faster """ from __future__ import division @@ -12,6 +14,8 @@ import numpy as np import inspect import itertools +import time +import logging class MetricsHost: """Keeps track of metrics and intra metric dependencies.""" @@ -19,7 +23,7 @@ class MetricsHost: def __init__(self): self.metrics = OrderedDict() - def register(self, fnc, deps='auto', name=None, helpstr=None, formatter=None): + def register(self, fnc, deps='auto', name=None, helpstr=None, formatter=None, fnc_m = None, deps_m = 'auto'): """Register a new metric. Params @@ -47,14 +51,22 @@ def register(self, fnc, deps='auto', name=None, helpstr=None, formatter=None): formatter: Format object, optional An optional default formatter when rendering metric results as string. I.e to render the result `0.35` as `35%` one would pass `{:.2%}.format` + fnc_m : Function or None, optional + Function that merges metric results. The number of arguments + is 1 + N, where N is the number of dependencies of the metric to be registered. + The order of the argument passed is `df, result_dep1, result_dep2, ...`. """ assert not fnc is None, 'No function given for metric {}'.format(name) if deps is None: deps = [] - elif deps is 'auto': - deps = inspect.getfullargspec(fnc).args[1:] # assumes dataframe as first argument + elif deps is 'auto': + if inspect.getfullargspec(fnc).defaults is not None: + k = - len(inspect.getfullargspec(fnc).defaults) + else: + k = len(inspect.getfullargspec(fnc).args) + deps = inspect.getfullargspec(fnc).args[1:k] # assumes dataframe as first argument if name is None: name = fnc.__name__ # Relies on meaningful function names, i.e don't use for lambdas @@ -62,11 +74,27 @@ def register(self, fnc, deps='auto', name=None, helpstr=None, formatter=None): if helpstr is None: helpstr = inspect.getdoc(fnc) if inspect.getdoc(fnc) else 'No description.' helpstr = ' '.join(helpstr.split()) - + if fnc_m is None and name+'_m' in globals(): + fnc_m = globals()[name+'_m'] + if fnc_m is not None: + if deps_m is None: + deps_m = [] + elif deps_m == 'auto': + if inspect.getfullargspec(fnc_m).defaults is not None: + k = - len(inspect.getfullargspec(fnc_m).defaults) + else: + k = len(inspect.getfullargspec(fnc_m).args) + deps_m = inspect.getfullargspec(fnc_m).args[1:k] # assumes dataframe as first argument + else: + deps_m = None + #print(name, 'merge function is None') + self.metrics[name] = { 'name' : name, 'fnc' : fnc, + 'fnc_m' : fnc_m, 'deps' : deps, + 'deps_m' : deps_m, 'help' : helpstr, 'formatter' : formatter } @@ -75,7 +103,7 @@ def register(self, fnc, deps='auto', name=None, helpstr=None, formatter=None): def names(self): """Returns the name identifiers of all registered metrics.""" return [v['name'] for v in self.metrics.values()] - + @property def formatters(self): """Returns the formatters for all metrics that have associated formatters.""" @@ -100,16 +128,18 @@ def list_metrics_markdown(self, include_deps=False): df_formatted = pd.concat([df_fmt, df]) return df_formatted.to_csv(sep="|", index=False) - def compute(self, df, metrics=None, return_dataframe=True, return_cached=False, name=None): + def compute(self, df, ana = None, metrics=None, return_dataframe=True, return_cached=False, name=None): """Compute metrics on the dataframe / accumulator. - + Params ------ df : MOTAccumulator or pandas.DataFrame The dataframe to compute the metrics on - + Kwargs ------ + ana: dict or None, optional + To cache results for fast computation. metrics : string, list of string or None, optional The identifiers of the metrics to be computed. This method will only compute the minimal set of necessary metrics to fullfill the request. @@ -122,7 +152,7 @@ def compute(self, df, metrics=None, return_dataframe=True, return_cached=False, When returning a pandas.DataFrame this is the index of the row containing the computed metric values. """ - + if isinstance(df, MOTAccumulator): df = df.events @@ -135,11 +165,16 @@ class DfMap : pass df_map = DfMap() df_map.full = df df_map.raw = df[df.Type == 'RAW'] - df_map.noraw = df[df.Type != 'RAW'] + df_map.noraw = df[(df.Type != 'RAW') & (df.Type != 'ASCEND') & (df.Type != 'TRANSFER') & (df.Type != 'MIGRATE')] + df_map.extra = df[df.Type != 'RAW'] cache = {} + options = {'ana': ana} for mname in metrics: - cache[mname] = self._compute(df_map, mname, cache, parent='summarize') + #st__ = time.time() + #print(mname, ' start') + cache[mname] = self._compute(df_map, mname, cache, options, parent='summarize') + #print('caling %s take '%mname, time.time()-st__) if name is None: name = 0 @@ -149,21 +184,64 @@ class DfMap : pass else: data = OrderedDict([(k, cache[k]) for k in metrics]) - print(metrics[0]) - print(data[metrics[0]]) - - return pd.DataFrame(data, index=[name]) if return_dataframe else data + ret = pd.DataFrame(data, index=[name]) if return_dataframe else data + return ret + + def compute_overall(self, partials, metrics = None, return_dataframe = True, return_cached = False, name = None): + """Compute overall metrics based on multiple results. + + Params + ------ + partials : list of metric results to combine overall + + Kwargs + ------ + metrics : string, list of string or None, optional + The identifiers of the metrics to be computed. This method will only + compute the minimal set of necessary metrics to fullfill the request. + If None is passed all registered metrics are computed. + return_dataframe : bool, optional + Return the result as pandas.DataFrame (default) or dict. + return_cached : bool, optional + If true all intermediate metrics required to compute the desired metrics are returned as well. + name : string, optional + When returning a pandas.DataFrame this is the index of the row containing + the computed metric values. + + Returns + ------- + df : pandas.DataFrame + A datafrom containing the metrics in columns and names in rows. + """ + if metrics is None: + metrics = self.names + elif isinstance(metrics, str): + metrics = [metrics] + cache = {} + + for mname in metrics: + cache[mname] = self._compute_overall(partials, mname, cache, parent = 'summarize') + + if name is None: + name = 0 + if return_cached: + data = cache + else: + data = OrderedDict([(k, cache[k]) for k in metrics]) + return pd.DataFrame(data, index=[name]) if return_dataframe else data - def compute_many(self, dfs, metrics=None, names=None, generate_overall=False): + def compute_many(self, dfs, anas=None, metrics=None, names=None, generate_overall=False): """Compute metrics on multiple dataframe / accumulators. - + Params ------ dfs : list of MOTAccumulator or list of pandas.DataFrame The data to compute metrics on. - + Kwargs ------ + anas: dict or None, optional + To cache results for fast computation. metrics : string, list of string or None, optional The identifiers of the metrics to be computed. This method will only compute the minimal set of necessary metrics to fullfill the request. @@ -183,33 +261,81 @@ def compute_many(self, dfs, metrics=None, names=None, generate_overall=False): """ assert names is None or len(names) == len(dfs) - + st = time.time() if names is None: names = range(len(dfs)) - + if anas is None: + anas = [None] * len(dfs) + partials = [ + self.compute(acc, + ana=analysis, + metrics=metrics, + name=name, + return_cached=True, + return_dataframe=False + ) + for acc, analysis, name in zip(dfs, anas, names)] + logging.info('partials: %.3f seconds.'%(time.time() - st)) + details = partials + #for detail in details: + # print(detail) + partials = [pd.DataFrame(OrderedDict([(k, i[k]) for k in metrics]), index=[name]) for i, name in zip(partials, names)] if generate_overall: - dfs += [MOTAccumulator.merge_event_dataframes(dfs)] - names += ['OVERALL'] - - partials = [self.compute(acc, metrics=metrics, name=name) for acc, name in zip(dfs, names)] + names = 'OVERALL' + # merged, infomap = MOTAccumulator.merge_event_dataframes(dfs, return_mappings = True) + # dfs = merged + # anas = MOTAccumulator.merge_analysis(anas, infomap) + # partials.append(self.compute(dfs, ana=anas, metrics=metrics, name=names)[0]) + partials.append(self.compute_overall(details, metrics=metrics, name=names)) + logging.info('mergeOverall: %.3f seconds.'%(time.time() - st)) return pd.concat(partials) - def _compute(self, df_map, name, cache, parent=None): + def _compute(self, df_map, name, cache, options, parent=None): """Compute metric and resolve dependencies.""" assert name in self.metrics, 'Cannot find metric {} required by {}.'.format(name, parent) - + already = cache.get(name, None) + if already is not None: + return already minfo = self.metrics[name] vals = [] for depname in minfo['deps']: v = cache.get(depname, None) if v is None: - v = cache[depname] = self._compute(df_map, depname, cache, parent=name) + #st_ = time.time() + #print(name, 'start calc dep ', depname) + v = cache[depname] = self._compute(df_map, depname, cache, options, parent=name) + #print(name, 'depends', depname, 'calculating %s take '%depname, time.time()-st_) + vals.append(v) + if inspect.getfullargspec(minfo['fnc']).defaults is None: + return minfo['fnc'](df_map, *vals) + else: + return minfo['fnc'](df_map, *vals, **options) + + def _compute_overall(self, partials, name, cache, parent = None): + assert name in self.metrics, 'Cannot find metric {} required by {}.'.format(name, parent) + #print('start computing %s'%name) + already = cache.get(name, None) + if already is not None: + return already + minfo = self.metrics[name] + vals = [] + for depname in minfo['deps_m']: + v = cache.get(depname, None) + if v is None: + #st_ = time.time() + #print(name, ' depends ', depname) + v = cache[depname] = self._compute_overall(partials, depname, cache, parent=name) + #print(name, 'depends', depname, 'calculating %s take '%depname, time.time()-st_) vals.append(v) - return minfo['fnc'](df_map, *vals) + assert minfo['fnc_m'] is not None, 'merge function for metric %s is None'%name + return minfo['fnc_m'](partials, *vals) + +simple_add_func = [] def num_frames(df): """Total number of frames.""" return df.full.index.get_level_values(0).unique().shape[0] +simple_add_func.append(num_frames) def obj_frequencies(df): """Total number of occurrences of individual objects over all frames.""" @@ -222,38 +348,62 @@ def pred_frequencies(df): def num_unique_objects(df, obj_frequencies): """Total number of unique object ids encountered.""" return len(obj_frequencies) +simple_add_func.append(num_unique_objects) def num_matches(df): """Total number matches.""" return df.noraw.Type.isin(['MATCH']).sum() +simple_add_func.append(num_matches) def num_switches(df): """Total number of track switches.""" return df.noraw.Type.isin(['SWITCH']).sum() +simple_add_func.append(num_switches) + +def num_transfer(df): + """Total number of track transfer.""" + return df.extra.Type.isin(['TRANSFER']).sum() +simple_add_func.append(num_transfer) + +def num_ascend(df): + """Total number of track ascend.""" + return df.extra.Type.isin(['ASCEND']).sum() +simple_add_func.append(num_ascend) + +def num_migrate(df): + """Total number of track migrate.""" + return df.extra.Type.isin(['MIGRATE']).sum() +simple_add_func.append(num_migrate) def num_false_positives(df): """Total number of false positives (false-alarms).""" return df.noraw.Type.isin(['FP']).sum() +simple_add_func.append(num_false_positives) def num_misses(df): """Total number of misses.""" return df.noraw.Type.isin(['MISS']).sum() +simple_add_func.append(num_misses) def num_detections(df, num_matches, num_switches): """Total number of detected objects including matches and switches.""" return num_matches + num_switches +simple_add_func.append(num_detections) def num_objects(df, obj_frequencies): """Total number of unique object appearances over all frames.""" return obj_frequencies.sum() +simple_add_func.append(num_objects) def num_predictions(df, pred_frequencies): """Total number of unique prediction appearances over all frames.""" return pred_frequencies.sum() +simple_add_func.append(num_predictions) def num_predictions(df): """Total number of unique prediction appearances over all frames.""" return df.noraw.HId.count() +simple_add_func.append(num_predictions) def track_ratios(df, obj_frequencies): """Ratio of assigned to total appearance count per unique object id.""" @@ -263,14 +413,17 @@ def track_ratios(df, obj_frequencies): def mostly_tracked(df, track_ratios): """Number of objects tracked for at least 80 percent of lifespan.""" return track_ratios[track_ratios >= 0.8].count() +simple_add_func.append(mostly_tracked) def partially_tracked(df, track_ratios): """Number of objects tracked between 20 and 80 percent of lifespan.""" return track_ratios[(track_ratios >= 0.2) & (track_ratios < 0.8)].count() +simple_add_func.append(partially_tracked) def mostly_lost(df, track_ratios): """Number of objects tracked less than 20 percent of lifespan.""" return track_ratios[track_ratios < 0.2].count() +simple_add_func.append(mostly_lost) def num_fragmentations(df, obj_frequencies): """Total number of switches from tracked to not tracked.""" @@ -287,33 +440,54 @@ def num_fragmentations(df, obj_frequencies): diffs = dfo.loc[first:last].Type.apply(lambda x: 1 if x == 'MISS' else 0).diff() fra += diffs[diffs == 1].count() return fra +simple_add_func.append(num_fragmentations) def motp(df, num_detections): """Multiple object tracker precision.""" return df.noraw['D'].sum() / num_detections +def motp_m(partials, num_detections): + res = 0 + for v in partials: + res += v['motp'] * v['num_detections'] + return res / num_detections + def mota(df, num_misses, num_switches, num_false_positives, num_objects): """Multiple object tracker accuracy.""" return 1. - (num_misses + num_switches + num_false_positives) / num_objects +def mota_m(partials, num_misses, num_switches, num_false_positives, num_objects): + return 1. - (num_misses + num_switches + num_false_positives) / num_objects + def precision(df, num_detections, num_false_positives): """Number of detected objects over sum of detected and false positives.""" return num_detections / (num_false_positives + num_detections) +def precision_m(partials, num_detections, num_false_positives): + return num_detections / (num_false_positives + num_detections) + def recall(df, num_detections, num_objects): """Number of detections over number of objects.""" return num_detections / num_objects -def id_global_assignment(df): - """ID measures: Global min-cost assignment for ID measures.""" +def recall_m(partials, num_detections, num_objects): + return num_detections / num_objects +def id_global_assignment(df, ana = None): + """ID measures: Global min-cost assignment for ID measures.""" + #st1 = time.time() oids = df.full['OId'].dropna().unique() hids = df.full['HId'].dropna().unique() hids_idx = dict((h,i) for i,h in enumerate(hids)) - - hcs = [len(df.raw[(df.raw.HId==h)].groupby(level=0)) for h in hids] - ocs = [len(df.raw[(df.raw.OId==o)].groupby(level=0)) for o in oids] - + #print('----'*2, '1', time.time()-st1) + if ana is None: + hcs = [len(df.raw[(df.raw.HId==h)].groupby(level=0)) for h in hids] + ocs = [len(df.raw[(df.raw.OId==o)].groupby(level=0)) for o in oids] + else: + hcs = [ana['hyp'][int(h)] for h in hids if h!='nan' and np.isfinite(float(h))] + ocs = [ana['obj'][int(o)] for o in oids if o!='nan' and np.isfinite(float(o))] + + #print('----'*2, '2', time.time()-st1) no = oids.shape[0] nh = hids.shape[0] @@ -321,11 +495,13 @@ def id_global_assignment(df): df = df.set_index(['OId','HId']) df = df.sort_index(level=[0,1]) + #print('----'*2, '3', time.time()-st1) fpmatrix = np.full((no+nh, no+nh), 0.) fnmatrix = np.full((no+nh, no+nh), 0.) fpmatrix[no:, :nh] = np.nan fnmatrix[:no, nh:] = np.nan + #print('----'*2, '4', time.time()-st1) for r, oc in enumerate(ocs): fnmatrix[r, :nh] = oc fnmatrix[r,nh+r] = oc @@ -334,6 +510,7 @@ def id_global_assignment(df): fpmatrix[:no, c] = hc fpmatrix[c+no,c] = hc + #print('----'*2, '5', time.time()-st1) for r, o in enumerate(oids): df_o = df.loc[o, 'D'].dropna() for h, ex in df_o.groupby(level=0).count().iteritems(): @@ -342,9 +519,13 @@ def id_global_assignment(df): fpmatrix[r,c] -= ex fnmatrix[r,c] -= ex + #print('----'*2, '6', time.time()-st1) + #print(fpmatrix.shape, fnmatrix.shape) costs = fpmatrix + fnmatrix + #print(costs.shape) rids, cids = linear_sum_assignment(costs) + #print('----'*2, '7', time.time()-st1) return { 'fpmatrix' : fpmatrix, 'fnmatrix' : fnmatrix, @@ -358,28 +539,77 @@ def idfp(df, id_global_assignment): """ID measures: Number of false positive matches after global min-cost matching.""" rids, cids = id_global_assignment['rids'], id_global_assignment['cids'] return id_global_assignment['fpmatrix'][rids, cids].sum() +simple_add_func.append(idfp) def idfn(df, id_global_assignment): """ID measures: Number of false negatives matches after global min-cost matching.""" rids, cids = id_global_assignment['rids'], id_global_assignment['cids'] return id_global_assignment['fnmatrix'][rids, cids].sum() +simple_add_func.append(idfn) def idtp(df, id_global_assignment, num_objects, idfn): """ID measures: Number of true positives matches after global min-cost matching.""" return num_objects - idfn +simple_add_func.append(idtp) def idp(df, idtp, idfp): """ID measures: global min-cost precision.""" return idtp / (idtp + idfp) +def idp_m(partials, idtp, idfp): + return idtp / (idtp + idfp) + def idr(df, idtp, idfn): """ID measures: global min-cost recall.""" return idtp / (idtp + idfn) +def idr_m(partials, idtp, idfn): + return idtp / (idtp + idfn) + def idf1(df, idtp, num_objects, num_predictions): """ID measures: global min-cost F1 score.""" return 2 * idtp / (num_objects + num_predictions) +def idf1_m(partials, idtp, num_objects, num_predictions): + return 2 * idtp / (num_objects + num_predictions) + +# def iou_sum(df): +# """Extra measures: sum IoU of all matches""" +# return (1 - df.noraw[(df.noraw.Type=='MATCH')|(df.noraw.Type=='SWITCH')].D).sum() + +# simple_add_func.append(iou_sum) + +# def siou_sum(df): +# """Extra measures: sum IoU of all matches""" +# return (1 - df.noraw[(df.noraw.Type=='SWITCH')].D).sum() + +# simple_add_func.append(siou_sum) + +# def avg_iou(df, iou_sum, num_matches, num_switches): +# """Extra measures: average IoU of all pairs""" +# return iou_sum / (num_matches + num_switches) + +# def avg_iou_m(partials, iou_sum, num_matches, num_switches): +# return iou_sum / (num_matches + num_switches) + +# def switch_iou(df, siou_sum, num_switches): +# """Extra measures: average IoU of all switches""" +# return siou_sum / (num_switches) + +# def switch_iou_m(partials, siou_sum, num_switches): +# return siou_sum / (num_switches) + +for one in simple_add_func: + name = one.__name__ + def getSimpleAdd(nm): + def simpleAddHolder(partials): + res = 0 + for v in partials: + res += v[nm] + return res + return simpleAddHolder + locals()[name+'_m'] = getSimpleAdd(name) + def create(): """Creates a MetricsHost and populates it with default metrics.""" m = MetricsHost() @@ -389,6 +619,9 @@ def create(): m.register(pred_frequencies, formatter='{:d}'.format) m.register(num_matches, formatter='{:d}'.format) m.register(num_switches, formatter='{:d}'.format) + m.register(num_transfer, formatter='{:d}'.format) + m.register(num_ascend, formatter='{:d}'.format) + m.register(num_migrate, formatter='{:d}'.format) m.register(num_false_positives, formatter='{:d}'.format) m.register(num_misses, formatter='{:d}'.format) m.register(num_detections, formatter='{:d}'.format) @@ -404,7 +637,7 @@ def create(): m.register(mota, formatter='{:.1%}'.format) m.register(precision, formatter='{:.1%}'.format) m.register(recall, formatter='{:.1%}'.format) - + m.register(id_global_assignment) m.register(idfp) m.register(idfn) @@ -413,7 +646,10 @@ def create(): m.register(idr, formatter='{:.1%}'.format) m.register(idf1, formatter='{:.1%}'.format) - + # m.register(iou_sum, formatter='{:.3f}'.format) + # m.register(siou_sum, formatter='{:.3f}'.format) + # m.register(avg_iou, formatter='{:.3f}'.format) + # m.register(switch_iou, formatter='{:.3f}'.format) return m motchallenge_metrics = [ @@ -431,6 +667,11 @@ def create(): 'num_switches', 'num_fragmentations', 'mota', - 'motp' + 'motp', + 'num_transfer', + 'num_ascend', + 'num_migrate', + # 'avg_iou', + # 'switch_iou', ] """A list of all metrics from MOTChallenge.""" diff --git a/motmetrics/mot.py b/motmetrics/mot.py index a0a11bf7..abb31f0e 100644 --- a/motmetrics/mot.py +++ b/motmetrics/mot.py @@ -1,7 +1,9 @@ """py-motmetrics - metrics for multiple object tracker (MOT) benchmarking. Christoph Heindl, 2017 -https://github.com/cheind/py-motmetrics +Toka, 2018 +Origin: https://github.com/cheind/py-motmetrics +TOKA make it faster """ import numpy as np @@ -13,46 +15,49 @@ class MOTAccumulator(object): """Manage tracking events. - - This class computes per-frame tracking events from a given set of object / hypothesis + + This class computes per-frame tracking events from a given set of object / hypothesis ids and pairwise distances. Indended usage import motmetrics as mm acc = mm.MOTAccumulator() acc.update(['a', 'b'], [0, 1, 2], dists, frameid=0) ... - acc.update(['d'], [6,10], other_dists, frameid=76) + acc.update(['d'], [6,10], other_dists, frameid=76) summary = mm.metrics.summarize(acc) print(mm.io.render_summary(summary)) Update is called once per frame and takes objects / hypothesis ids and a pairwise distance - matrix between those (see distances module for support). Per frame max(len(objects), len(hypothesis)) + matrix between those (see distances module for support). Per frame max(len(objects), len(hypothesis)) events are generated. Each event type is one of the following - `'MATCH'` a match between a object and hypothesis was found - - `'SWITCH'` a match between a object and hypothesis was found but differs from previous assignment + - `'SWITCH'` a match between a object and hypothesis was found but differs from previous assignment (hypothesisid != previous) - `'MISS'` no match for an object was found - `'FP'` no match for an hypothesis was found (spurious detections) - `'RAW'` events corresponding to raw input - + - `'TRANSFER'` a match between a object and hypothesis was found but differs from previous assignment (objectid != previous) + - `'ASCEND'` a match between a object and hypothesis was found but differs from previous assignment (hypothesisid is new) + - `'MIGRATE'` a match between a object and hypothesis was found but differs from previous assignment (objectid is new) + Events are tracked in a pandas Dataframe. The dataframe is hierarchically indexed by (`FrameId`, `EventId`), where `FrameId` is either provided during the call to `update` or auto-incremented when `auto_id` is set true during construction of MOTAccumulator. `EventId` is auto-incremented. The dataframe has the following - columns + columns - `Type` one of `('MATCH', 'SWITCH', 'MISS', 'FP', 'RAW')` - `OId` object id or np.nan when `'FP'` or `'RAW'` and object is not present - `HId` hypothesis id or np.nan when `'MISS'` or `'RAW'` and hypothesis is not present - `D` distance or np.nan when `'FP'` or `'MISS'` or `'RAW'` and either object/hypothesis is absent - - From the events and associated fields the entire tracking history can be recovered. Once the accumulator + + From the events and associated fields the entire tracking history can be recovered. Once the accumulator has been populated with per-frame data use `metrics.summarize` to compute statistics. See `metrics.compute_metrics` for a list of metrics computed. References ---------- - 1. Bernardin, Keni, and Rainer Stiefelhagen. "Evaluating multiple object tracking performance: the CLEAR MOT metrics." + 1. Bernardin, Keni, and Rainer Stiefelhagen. "Evaluating multiple object tracking performance: the CLEAR MOT metrics." EURASIP Journal on Image and Video Processing 2008.1 (2008): 1-10. 2. Milan, Anton, et al. "Mot16: A benchmark for multi-object tracking." arXiv preprint arXiv:1603.00831 (2016). - 3. Li, Yuan, Chang Huang, and Ram Nevatia. "Learning to associate: Hybridboosted multi-target tracker for crowded scene." + 3. Li, Yuan, Chang Huang, and Ram Nevatia. "Learning to associate: Hybridboosted multi-target tracker for crowded scene." Computer Vision and Pattern Recognition, 2009. CVPR 2009. IEEE Conference on. IEEE, 2009. """ @@ -68,10 +73,10 @@ def __init__(self, auto_id=False, max_switch_time=float('inf')): false also results in an error. max_switch_time : scalar, optional - Allows specifying an upper bound on the timespan an unobserved but - tracked object is allowed to generate track switch events. Useful if groundtruth - objects leaving the field of view keep their ID when they reappear, - but your tracker is not capable of recognizing this (resulting in + Allows specifying an upper bound on the timespan an unobserved but + tracked object is allowed to generate track switch events. Useful if groundtruth + objects leaving the field of view keep their ID when they reappear, + but your tracker is not capable of recognizing this (resulting in track switch events). The default is that there is no upper bound on the timespan. In units of frame timestamps. When using auto_id in units of count. @@ -79,7 +84,7 @@ def __init__(self, auto_id=False, max_switch_time=float('inf')): self.auto_id = auto_id self.max_switch_time = max_switch_time - self.reset() + self.reset() def reset(self): """Reset the accumulator to empty state.""" @@ -87,40 +92,43 @@ def reset(self): self._events = [] self._indices = [] #self.events = MOTAccumulator.new_event_dataframe() - self.m = {} # Pairings up to current timestamp + self.m = {} # Pairings up to current timestamp + self.res_m = {} # Result pairings up to now self.last_occurrence = {} # Tracks most recent occurance of object + self.last_match = {} # Tracks most recent match of object + self.hypHistory = {} self.dirty_events = True self.cached_events_df = None - def update(self, oids, hids, dists, frameid=None): + def update(self, oids, hids, dists, frameid=None, vf=''): """Updates the accumulator with frame specific objects/detections. This method generates events based on the following algorithm [1]: 1. Try to carry forward already established tracks. If any paired object / hypothesis - from previous timestamps are still visible in the current frame, create a 'MATCH' + from previous timestamps are still visible in the current frame, create a 'MATCH' event between them. 2. For the remaining constellations minimize the total object / hypothesis distance error (Kuhn-Munkres algorithm). If a correspondence made contradicts a previous match create a 'SWITCH' else a 'MATCH' event. 3. Create 'MISS' events for all remaining unassigned objects. 4. Create 'FP' events for all remaining unassigned hypotheses. - + Params ------ - oids : N array + oids : N array Array of object ids. - hids : M array + hids : M array Array of hypothesis ids. dists: NxM array Distance matrix. np.nan values to signal do-not-pair constellations. - See `distances` module for support methods. + See `distances` module for support methods. Kwargs ------ frameId : id Unique frame id. Optional when MOTAccumulator.auto_id is specified during construction. - + vf: file to log details Returns ------- frame_events : pd.DataFrame @@ -128,16 +136,16 @@ def update(self, oids, hids, dists, frameid=None): References ---------- - 1. Bernardin, Keni, and Rainer Stiefelhagen. "Evaluating multiple object tracking performance: the CLEAR MOT metrics." + 1. Bernardin, Keni, and Rainer Stiefelhagen. "Evaluating multiple object tracking performance: the CLEAR MOT metrics." EURASIP Journal on Image and Video Processing 2008.1 (2008): 1-10. """ - + self.dirty_events = True oids = ma.array(oids, mask=np.zeros(len(oids))) - hids = ma.array(hids, mask=np.zeros(len(hids))) + hids = ma.array(hids, mask=np.zeros(len(hids))) dists = np.atleast_2d(dists).astype(float).reshape(oids.shape[0], hids.shape[0]).copy() - if frameid is None: + if frameid is None: assert self.auto_id, 'auto-id is not enabled' if len(self._indices) > 0: frameid = self._indices[-1][0] + 1 @@ -145,14 +153,14 @@ def update(self, oids, hids, dists, frameid=None): frameid = 0 else: assert not self.auto_id, 'Cannot provide frame id when auto-id is enabled' - + eid = count() # 0. Record raw events no = len(oids) nh = len(hids) - + if no * nh > 0: for i in range(no): for j in range(nh): @@ -161,66 +169,98 @@ def update(self, oids, hids, dists, frameid=None): elif no == 0: for i in range(nh): self._indices.append((frameid, next(eid))) - self._events.append(['RAW', np.nan, hids[i], np.nan]) + self._events.append(['RAW', np.nan, hids[i], np.nan]) elif nh == 0: for i in range(no): self._indices.append((frameid, next(eid))) self._events.append(['RAW', oids[i], np.nan, np.nan]) - if oids.size * hids.size > 0: + if oids.size * hids.size > 0: # 1. Try to re-establish tracks from previous correspondences for i in range(oids.shape[0]): if not oids[i] in self.m: continue - hprev = self.m[oids[i]] - j, = np.where(hids==hprev) + hprev = self.m[oids[i]] + j, = np.where(hids==hprev) if j.shape[0] == 0: continue j = j[0] if np.isfinite(dists[i,j]): + o = oids[i] + h = hids[j] oids[i] = ma.masked hids[j] = ma.masked self.m[oids.data[i]] = hids.data[j] - + self._indices.append((frameid, next(eid))) self._events.append(['MATCH', oids.data[i], hids.data[j], dists[i, j]]) + self.last_match[o] = frameid + self.hypHistory[h] = frameid # 2. Try to remaining objects/hypotheses dists[oids.mask, :] = np.nan dists[:, hids.mask] = np.nan - + rids, cids = linear_sum_assignment(dists) - for i, j in zip(rids, cids): + for i, j in zip(rids, cids): if not np.isfinite(dists[i,j]): continue - + o = oids[i] h = hids.data[j] is_switch = o in self.m and \ self.m[o] != h and \ abs(frameid - self.last_occurrence[o]) <= self.max_switch_time - cat = 'SWITCH' if is_switch else 'MATCH' + cat1 = 'SWITCH' if is_switch else 'MATCH' + if cat1=='SWITCH': + if h not in self.hypHistory: + subcat = 'ASCEND' + self._indices.append((frameid, next(eid))) + self._events.append([subcat, oids.data[i], hids.data[j], dists[i, j]]) + is_transfer = h in self.res_m and \ + self.res_m[h] != o #and \ + # abs(frameid - self.last_occurrence[o]) <= self.max_switch_time # ignore this condition temporarily + cat2 = 'TRANSFER' if is_transfer else 'MATCH' + if cat2=='TRANSFER': + if o not in self.last_match: + subcat = 'MIGRATE' + self._indices.append((frameid, next(eid))) + self._events.append([subcat, oids.data[i], hids.data[j], dists[i, j]]) + self._indices.append((frameid, next(eid))) + self._events.append([cat2, oids.data[i], hids.data[j], dists[i, j]]) + if vf!='' and (cat1!='MATCH' or cat2!='MATCH'): + if cat1=='SWITCH': + vf.write('%s %d %d %d %d %d\n'%(subcat[:2], o, self.last_match[o], self.m[o], frameid, h)) + if cat2=='TRANSFER': + vf.write('%s %d %d %d %d %d\n'%(subcat[:2], h, self.hypHistory[h], self.res_m[h], frameid, o)) + self.hypHistory[h] = frameid + self.last_match[o] = frameid self._indices.append((frameid, next(eid))) - self._events.append([cat, oids.data[i], hids.data[j], dists[i, j]]) + self._events.append([cat1, oids.data[i], hids.data[j], dists[i, j]]) oids[i] = ma.masked hids[j] = ma.masked self.m[o] = h + self.res_m[h] = o # 3. All remaining objects are missed for o in oids[~oids.mask]: self._indices.append((frameid, next(eid))) self._events.append(['MISS', o, np.nan, np.nan]) - + if vf!='': + vf.write('FN %d %d\n'%(frameid, o)) + # 4. All remaining hypotheses are false alarms for h in hids[~hids.mask]: self._indices.append((frameid, next(eid))) self._events.append(['FP', np.nan, h, np.nan]) + if vf!='': + vf.write('FP %d %d\n'%(frameid, h)) # 5. Update occurance state - for o in oids.data: + for o in oids.data: self.last_occurrence[o] = frameid return frameid @@ -231,7 +271,7 @@ def events(self): self.cached_events_df = MOTAccumulator.new_event_dataframe_with_data(self._indices, self._events) self.dirty_events = False return self.cached_events_df - + @property def mot_events(self): df = self.events @@ -241,13 +281,13 @@ def mot_events(self): def new_event_dataframe(): """Create a new DataFrame for event tracking.""" idx = pd.MultiIndex(levels=[[],[]], labels=[[],[]], names=['FrameId','Event']) - cats = pd.Categorical([], categories=['RAW', 'FP', 'MISS', 'SWITCH', 'MATCH']) + cats = pd.Categorical([], categories=['RAW', 'FP', 'MISS', 'SWITCH', 'MATCH', 'TRANSFER', 'ASCEND', 'MIGRATE']) df = pd.DataFrame( OrderedDict([ ('Type', pd.Series(cats)), # Type of event. One of FP (false positive), MISS, SWITCH, MATCH ('OId', pd.Series(dtype=object)), # Object ID or -1 if FP. Using float as missing values will be converted to NaN anyways. ('HId', pd.Series(dtype=object)), # Hypothesis ID or NaN if MISS. Using float as missing values will be converted to NaN anyways. - ('D', pd.Series(dtype=float)), # Distance or NaN when FP or MISS + ('D', pd.Series(dtype=float)), # Distance or NaN when FP or MISS ]), index=idx ) @@ -256,42 +296,57 @@ def new_event_dataframe(): @staticmethod def new_event_dataframe_with_data(indices, events): """Create a new DataFrame filled with data. - + Params ------ indices: list list of tuples (frameid, eventid) events: list list of events where each event is a list containing - 'Type', 'OId', HId', 'D' + 'Type', 'OId', HId', 'D' """ tevents = list(zip(*events)) - raw_type = pd.Categorical(tevents[0], categories=['RAW', 'FP', 'MISS', 'SWITCH', 'MATCH'], ordered=False) + raw_type = pd.Categorical(tevents[0], categories=['RAW', 'FP', 'MISS', 'SWITCH', 'MATCH', 'TRANSFER', 'ASCEND', 'MIGRATE'], ordered=False) series = [ pd.Series(raw_type, name='Type'), pd.Series(tevents[1], dtype=object, name='OId'), pd.Series(tevents[2], dtype=object, name='HId'), pd.Series(tevents[3], dtype=float, name='D') ] - + idx = pd.MultiIndex.from_tuples(indices, names=['FrameId','Event']) df = pd.concat(series, axis=1) df.index = idx return df - + @staticmethod + def merge_analysis(anas, infomap): + res = {'hyp':{}, 'obj':{}} + mapp = {'hyp': 'hid_map', 'obj':'oid_map'} + for ana, infom in zip(anas, infomap): + if ana is None: return None + for t in ana.keys(): + which = mapp[t] + if np.nan in infom[which]: + res[t][int(infom[which][np.nan])] = 0 + if 'nan' in infom[which]: + res[t][int(infom[which]['nan'])] = 0 + for _id, cnt in ana[t].items(): + if _id not in infom[which]: _id = str(_id) + res[t][int(infom[which][_id])] = cnt + return res @staticmethod def merge_event_dataframes(dfs, update_frame_indices=True, update_oids=True, update_hids=True, return_mappings=False): """Merge dataframes. - + Params ------ dfs : list of pandas.DataFrame or MotAccumulator A list of event containers to merge - + Kwargs ------ update_frame_indices : boolean, optional @@ -306,7 +361,7 @@ def merge_event_dataframes(dfs, update_frame_indices=True, update_oids=True, upd Returns ------- df : pandas.DataFrame - Merged event data frame + Merged event data frame """ mapping_infos = [] @@ -321,7 +376,7 @@ def merge_event_dataframes(dfs, update_frame_indices=True, update_oids=True, upd copy = df.copy() infos = {} - + # Update index if update_frame_indices: next_frame_id = max(r.index.get_level_values(0).max()+1, r.index.get_level_values(0).unique().shape[0]) @@ -331,20 +386,20 @@ def merge_event_dataframes(dfs, update_frame_indices=True, update_oids=True, upd infos['frame_offset'] = next_frame_id # Update object / hypothesis ids - if update_oids: + if update_oids: oid_map = dict([oid, str(next(new_oid))] for oid in copy['OId'].dropna().unique()) copy['OId'] = copy['OId'].map(lambda x: oid_map[x], na_action='ignore') infos['oid_map'] = oid_map - + if update_hids: hid_map = dict([hid, str(next(new_hid))] for hid in copy['HId'].dropna().unique()) copy['HId'] = copy['HId'].map(lambda x: hid_map[x], na_action='ignore') infos['hid_map'] = hid_map - + r = r.append(copy) mapping_infos.append(infos) if return_mappings: return r, mapping_infos - else: - return r \ No newline at end of file + else: + return r diff --git a/motmetrics/preprocess.py b/motmetrics/preprocess.py new file mode 100644 index 00000000..5c54b844 --- /dev/null +++ b/motmetrics/preprocess.py @@ -0,0 +1,65 @@ +"""py-motmetrics - metrics for multiple object tracker (MOT) benchmarking. + +Toka, 2018 +Origin: https://github.com/cheind/py-motmetrics +Extended: +""" + +import numpy as np +import pandas as pd +from configparser import ConfigParser +from motmetrics.lap import linear_sum_assignment +import motmetrics.distances as mmd +import time +import logging + +def preprocessResult(res, gt, inifile): + st = time.time() + labels = ['ped', # 1 + 'person_on_vhcl', # 2 + 'car', # 3 + 'bicycle', # 4 + 'mbike', # 5 + 'non_mot_vhcl', # 6 + 'static_person', # 7 + 'distractor', # 8 + 'occluder', # 9 + 'occluder_on_grnd', #10 + 'occluder_full', # 11 + 'reflection', # 12 + 'crowd' # 13 + ] + distractors_ = ['person_on_vhcl','static_person','distractor','reflection'] + distractors = {i+1 : x in distractors_ for i,x in enumerate(labels)} + for i in distractors_: + distractors[i] = 1 + seqIni = ConfigParser() + seqIni.read(inifile, encoding='utf8') + F = int(seqIni['Sequence']['seqLength']) + todrop = [] + for t in range(1,F+1): + if t not in res.index or t not in gt.index: continue + #st = time.time() + resInFrame = res.loc[t] + N = len(resInFrame) + + GTInFrame = gt.loc[t] + Ngt = len(GTInFrame) + A = GTInFrame[['X','Y','Width','Height']].values + B = resInFrame[['X','Y','Width','Height']].values + disM = mmd.iou_matrix(A, B, max_iou = 0.5) + #en = time.time() + #print('----', 'disM', en - st) + le, ri = linear_sum_assignment(disM) + flags = [1 if distractors[it['ClassId']] or it['Visibility']<0. else 0 for i,(k,it) in enumerate(GTInFrame.iterrows())] + hid = [k for k,it in resInFrame.iterrows()] + for i, j in zip(le, ri): + if not np.isfinite(disM[i, j]): + continue + if flags[i]: + todrop.append((t, hid[j])) + #en = time.time() + #print('Frame %d: '%t, en - st) + ret = res.drop(labels=todrop) + logging.info('Preprocess take %.3f seconds and remove %d boxes.'%(time.time() - st, len(todrop))) + return ret diff --git a/motmetrics/tests/test_metrics.py b/motmetrics/tests/test_metrics.py index 90197200..e72683b4 100644 --- a/motmetrics/tests/test_metrics.py +++ b/motmetrics/tests/test_metrics.py @@ -92,6 +92,38 @@ def test_mota_motp(): assert metr['mota'] == approx(1. - (2 + 2 + 2) / 8) assert metr['motp'] == approx(11.1 / 6) +def test_ids(): + acc = mm.MOTAccumulator() + + # No data + acc.update([], [], [], frameid=0) + # Match + acc.update([1, 2], ['a', 'b'], [[1, 0], [0, 1]], frameid=1) + # Switch also Transfer + acc.update([1, 2], ['a', 'b'], [[0.4, np.nan], [np.nan, 0.4]], frameid=2) + # Match + acc.update([1, 2], ['a', 'b'], [[0, 1], [1, 0]], frameid=3) + # Ascend (switch) + acc.update([1, 2], ['b', 'c'], [[1, 0], [0.4, 0.7]], frameid=4) + # Migrate (transfer) + acc.update([1, 3], ['b', 'c'], [[1, 0], [0.4, 0.7]], frameid=5) + # No data + acc.update([], [], [], frameid=6) + + mh = mm.metrics.create() + metr = mh.compute(acc, metrics=['motp', 'mota', 'num_predictions', 'num_transfer', 'num_ascend', 'num_migrate'], return_dataframe=False, return_cached=True) + assert metr['num_matches'] == 7 + assert metr['num_false_positives'] == 0 + assert metr['num_misses'] == 0 + assert metr['num_switches'] == 3 + assert metr['num_transfer'] == 3 + assert metr['num_ascend'] == 1 + assert metr['num_migrate'] == 1 + assert metr['num_detections'] == 10 + assert metr['num_objects'] == 10 + assert metr['num_predictions'] == 10 + assert metr['mota'] == approx(1. - (0 + 0 + 3) / 10) + assert metr['motp'] == approx(1.6 / 10) def test_correct_average(): # Tests what is being depicted in figure 3 of 'Evaluating MOT Performance' @@ -134,11 +166,11 @@ def compute_motchallenge(dname): print() print(mm.io.render_summary(summary, namemap=mm.io.motchallenge_metric_names, formatters=mh.formatters)) - + # assert ((summary['num_transfer'] - summary['num_migrate']) == (summary['num_switches'] - summary['num_ascend'])).all() # False assertion + summary = summary[mm.metrics.motchallenge_metrics[:15]] expected = pd.DataFrame([ [0.557659, 0.729730, 0.451253, 0.582173, 0.941441, 8.0, 1, 6, 1, 13, 150, 7, 7, 0.526462, 0.277201], [0.644619, 0.819760, 0.531142, 0.608997, 0.939920, 10.0, 5, 4, 1, 45, 452, 7, 6, 0.564014, 0.345904], [0.624296, 0.799176, 0.512211, 0.602640, 0.940268, 18.0, 6, 10, 2, 58, 602, 14, 13, 0.555116, 0.330177], ]) - - np.testing.assert_allclose(summary, expected, atol=1e-3) \ No newline at end of file + np.testing.assert_allclose(summary, expected, atol=1e-3) diff --git a/motmetrics/tests/test_mot.py b/motmetrics/tests/test_mot.py index c24eb11f..5a8695ec 100644 --- a/motmetrics/tests/test_mot.py +++ b/motmetrics/tests/test_mot.py @@ -43,8 +43,10 @@ def test_events(): expect.loc[(3, 1), :] = ['RAW', 1, 'b', np.nan] expect.loc[(3, 2), :] = ['RAW', 2, 'a', np.nan] expect.loc[(3, 3), :] = ['RAW', 2, 'b', 0.1] - expect.loc[(3, 4), :] = ['SWITCH', 1, 'a', 0.2] - expect.loc[(3, 5), :] = ['SWITCH', 2, 'b', 0.1] + expect.loc[(3, 4), :] = ['TRANSFER', 1, 'a', 0.2] + expect.loc[(3, 5), :] = ['SWITCH', 1, 'a', 0.2] + expect.loc[(3, 6), :] = ['TRANSFER', 2, 'b', 0.1] + expect.loc[(3, 7), :] = ['SWITCH', 2, 'b', 0.1] expect.loc[(4, 0), :] = ['RAW', 1, 'a', 5.] expect.loc[(4, 1), :] = ['RAW', 1, 'b', 1.] @@ -64,14 +66,14 @@ def test_max_switch_time(): frameid = acc.update([1, 2], ['a', 'b'], [[0.5, np.nan], [np.nan, 0.5]], frameid=2) # 1->b, 2->a df = acc.events.loc[frameid] - assert ((df.Type == 'SWITCH') | (df.Type == 'RAW')).all() + assert ((df.Type == 'SWITCH') | (df.Type == 'RAW') | (df.Type == 'TRANSFER')).all() acc = mm.MOTAccumulator(max_switch_time=1) acc.update([1, 2], ['a', 'b'], [[1, 0.5], [0.3, 1]], frameid=1) # 1->a, 2->b frameid = acc.update([1, 2], ['a', 'b'], [[0.5, np.nan], [np.nan, 0.5]], frameid=5) # Later frame 1->b, 2->a df = acc.events.loc[frameid] - assert ((df.Type == 'MATCH') | (df.Type == 'RAW')).all() + assert ((df.Type == 'MATCH') | (df.Type == 'RAW') | (df.Type == 'TRANSFER')).all() def test_auto_id(): acc = mm.MOTAccumulator(auto_id=True) @@ -141,8 +143,10 @@ def test_merge_dataframes(): expect.loc[(3, 1), :] = ['RAW', mappings[0]['oid_map'][1], mappings[0]['hid_map']['b'], np.nan] expect.loc[(3, 2), :] = ['RAW', mappings[0]['oid_map'][2], mappings[0]['hid_map']['a'], np.nan] expect.loc[(3, 3), :] = ['RAW', mappings[0]['oid_map'][2], mappings[0]['hid_map']['b'], 0.1] - expect.loc[(3, 4), :] = ['SWITCH', mappings[0]['oid_map'][1], mappings[0]['hid_map']['a'], 0.2] - expect.loc[(3, 5), :] = ['SWITCH', mappings[0]['oid_map'][2], mappings[0]['hid_map']['b'], 0.1] + expect.loc[(3, 4), :] = ['TRANSFER', mappings[0]['oid_map'][1], mappings[0]['hid_map']['a'], 0.2] + expect.loc[(3, 5), :] = ['SWITCH', mappings[0]['oid_map'][1], mappings[0]['hid_map']['a'], 0.2] + expect.loc[(3, 6), :] = ['TRANSFER', mappings[0]['oid_map'][2], mappings[0]['hid_map']['b'], 0.1] + expect.loc[(3, 7), :] = ['SWITCH', mappings[0]['oid_map'][2], mappings[0]['hid_map']['b'], 0.1] # Merge duplication expect.loc[(4, 0), :] = ['RAW', np.nan, mappings[1]['hid_map']['a'], np.nan] @@ -166,8 +170,10 @@ def test_merge_dataframes(): expect.loc[(7, 1), :] = ['RAW', mappings[1]['oid_map'][1], mappings[1]['hid_map']['b'], np.nan] expect.loc[(7, 2), :] = ['RAW', mappings[1]['oid_map'][2], mappings[1]['hid_map']['a'], np.nan] expect.loc[(7, 3), :] = ['RAW', mappings[1]['oid_map'][2], mappings[1]['hid_map']['b'], 0.1] - expect.loc[(7, 4), :] = ['SWITCH', mappings[1]['oid_map'][1], mappings[1]['hid_map']['a'], 0.2] - expect.loc[(7, 5), :] = ['SWITCH', mappings[1]['oid_map'][2], mappings[1]['hid_map']['b'], 0.1] + expect.loc[(7, 4), :] = ['TRANSFER', mappings[1]['oid_map'][1], mappings[1]['hid_map']['a'], 0.2] + expect.loc[(7, 5), :] = ['SWITCH', mappings[1]['oid_map'][1], mappings[1]['hid_map']['a'], 0.2] + expect.loc[(7, 6), :] = ['TRANSFER', mappings[1]['oid_map'][2], mappings[1]['hid_map']['b'], 0.1] + expect.loc[(7, 7), :] = ['SWITCH', mappings[1]['oid_map'][2], mappings[1]['hid_map']['b'], 0.1] from pandas.util.testing import assert_frame_equal assert_frame_equal(r, expect) diff --git a/motmetrics/utils.py b/motmetrics/utils.py index 961c624c..6651e1b9 100644 --- a/motmetrics/utils.py +++ b/motmetrics/utils.py @@ -1,7 +1,9 @@ """py-motmetrics - metrics for multiple object tracker (MOT) benchmarking. Christoph Heindl, 2017 +Toka, 2018 https://github.com/cheind/py-motmetrics +TOKA EXTENDED THIS FILE. """ import pandas as pd @@ -9,6 +11,7 @@ from .mot import MOTAccumulator from .distances import iou_matrix, norm2squared_matrix +from .preprocess import preprocessResult def compare_to_groundtruth(gt, dt, dist='iou', distfields=['X', 'Y', 'Width', 'Height'], distth=0.5): """Compare groundtruth and detector results. @@ -69,4 +72,87 @@ def compute_euc(a, b): acc.update(oids, hids, dists, frameid=fid) - return acc \ No newline at end of file + return acc + +def CLEAR_MOT_M(gt, dt, inifile, dist='iou', distfields=['X', 'Y', 'Width', 'Height'], distth=0.5, include_all = False, vflag = ''): + """Compare groundtruth and detector results. + + This method assumes both results are given in terms of DataFrames with at least the following fields + - `FrameId` First level index used for matching ground-truth and test frames. + - `Id` Secondary level index marking available object / hypothesis ids + + Depending on the distance to be used relevant distfields need to be specified. + + Params + ------ + gt : pd.DataFrame + Dataframe for ground-truth + test : pd.DataFrame + Dataframe for detector results + + Kwargs + ------ + dist : str, optional + String identifying distance to be used. Defaults to intersection over union. + distfields: array, optional + Fields relevant for extracting distance information. Defaults to ['X', 'Y', 'Width', 'Height'] + distth: float, optional + Maximum tolerable distance. Pairs exceeding this threshold are marked 'do-not-pair'. + """ + + def compute_iou(a, b): + return iou_matrix(a, b, max_iou=distth) + + def compute_euc(a, b): + return norm2squared_matrix(a, b, max_d2=distth) + + compute_dist = compute_iou if dist.upper() == 'IOU' else compute_euc + + acc = MOTAccumulator() + #import time + #print('preprocess start.') + #pst = time.time() + dt = preprocessResult(dt, gt, inifile) + #pen = time.time() + #print('preprocess take ', pen - pst) + if include_all: + gt = gt[gt['Confidence'] >= 0.99] + else: + gt = gt[ (gt['Confidence'] >= 0.99) & (gt['ClassId'] == 1) ] + # We need to account for all frames reported either by ground truth or + # detector. In case a frame is missing in GT this will lead to FPs, in + # case a frame is missing in detector results this will lead to FNs. + allframeids = gt.index.union(dt.index).levels[0] + analysis = {'hyp':{}, 'obj':{}} + for fid in allframeids: + #st = time.time() + oids = np.empty(0) + hids = np.empty(0) + dists = np.empty((0,0)) + + if fid in gt.index: + fgt = gt.loc[fid] + oids = fgt.index.values + for oid in oids: + oid = int(oid) + if oid not in analysis['obj']: + analysis['obj'][oid] = 0 + analysis['obj'][oid] += 1 + + if fid in dt.index: + fdt = dt.loc[fid] + hids = fdt.index.values + for hid in hids: + hid = int(hid) + if hid not in analysis['hyp']: + analysis['hyp'][hid] = 0 + analysis['hyp'][hid] += 1 + + if oids.shape[0] > 0 and hids.shape[0] > 0: + dists = compute_dist(fgt[distfields].values, fdt[distfields].values) + + acc.update(oids, hids, dists, frameid=fid, vf = vflag) + #en = time.time() + #print(fid, ' time ', en - st) + + return acc, analysis diff --git a/seqmaps/example.txt b/seqmaps/example.txt new file mode 100644 index 00000000..b42b7a9b --- /dev/null +++ b/seqmaps/example.txt @@ -0,0 +1,3 @@ +name +MOT16-02 +MOT16-04 diff --git a/test.sh.example b/test.sh.example new file mode 100644 index 00000000..aa73e5c2 --- /dev/null +++ b/test.sh.example @@ -0,0 +1 @@ +python3 -u -m motmetrics.apps.evaluateTracking ./gt_dir/ ./res_dir/ ./seqmaps/example.txt