From 3dc9ea1621f644d5dd3a4242f5c37e0fbb474b37 Mon Sep 17 00:00:00 2001 From: Urwa Muaz <43106180+muaz-urwa@users.noreply.github.com> Date: Thu, 31 Oct 2019 00:53:33 -0400 Subject: [PATCH] adding suport for reading .mat and .xml format annotations given by UA-DETRAC challenge. (#53) * version bump * fix realease readme * adding suport for reading detrac .mat and .xml format * added unit tests for detrac data loaders * correcting the top,left co-ordinate derivation in derac mat file loader --- motmetrics/apps/eval_detrac.py | 105 +++++++++++++++++++++++ motmetrics/data/iotest/detrac.mat | Bin 0 -> 2752 bytes motmetrics/data/iotest/detrac.xml | 41 +++++++++ motmetrics/io.py | 137 +++++++++++++++++++++++++++++- motmetrics/tests/test_io.py | 26 +++++- requirements.txt | 1 + 6 files changed, 308 insertions(+), 2 deletions(-) create mode 100644 motmetrics/apps/eval_detrac.py create mode 100644 motmetrics/data/iotest/detrac.mat create mode 100644 motmetrics/data/iotest/detrac.xml diff --git a/motmetrics/apps/eval_detrac.py b/motmetrics/apps/eval_detrac.py new file mode 100644 index 00000000..c3b53d66 --- /dev/null +++ b/motmetrics/apps/eval_detrac.py @@ -0,0 +1,105 @@ +"""py-motmetrics - metrics for multiple object tracker (MOT) benchmarking. + +Christoph Heindl, 2017 +https://github.com/cheind/py-motmetrics + +Author: Urwa Muaz +""" + +import argparse +import glob +import os +import logging +import motmetrics as mm +import pandas as pd +from collections import OrderedDict +from pathlib import Path + +def parse_args(): + parser = argparse.ArgumentParser(description=""" +Compute metrics for trackers using DETRAC challenge ground-truth data. + +Files +----- +Ground truth files can be in .XML format or .MAT format as provided by http://detrac-db.rit.albany.edu/download + +Test Files for the challenge are reuired to be in MOTchallenge format, they have to comply with the format described in + +Milan, Anton, et al. +"Mot16: A benchmark for multi-object tracking." +arXiv preprint arXiv:1603.00831 (2016). +https://motchallenge.net/ + +Directory Structure +--------- + +Layout for ground truth data + /.txt + /.txt + ... + + OR + /.mat + /.mat + ... + +Layout for test data + /.txt + /.txt + ... + +Sequences of ground truth and test will be matched according to the `` +string.""", formatter_class=argparse.RawTextHelpFormatter) + + parser.add_argument('groundtruths', type=str, help='Directory containing ground truth files.') + parser.add_argument('tests', type=str, help='Directory containing tracker result files') + parser.add_argument('--loglevel', type=str, help='Log level', default='info') + parser.add_argument('--gtfmt', type=str, help='Groundtruth data format', default='detrac-xml') + parser.add_argument('--tsfmt', type=str, help='Test data format', default='mot15-2D') + parser.add_argument('--solver', type=str, help='LAP solver to use') + return parser.parse_args() + +def compare_dataframes(gts, ts): + accs = [] + names = [] + for k, tsacc in ts.items(): + if k in gts: + logging.info('Comparing {}...'.format(k)) + accs.append(mm.utils.compare_to_groundtruth(gts[k], tsacc, 'iou', distth=0.5)) + names.append(k) + else: + logging.warning('No ground truth for {}, skipping.'.format(k)) + + return accs, names + +if __name__ == '__main__': + + args = parse_args() + + loglevel = getattr(logging, args.loglevel.upper(), None) + if not isinstance(loglevel, int): + raise ValueError('Invalid log level: {} '.format(args.loglevel)) + logging.basicConfig(level=loglevel, format='%(asctime)s %(levelname)s - %(message)s', datefmt='%I:%M:%S') + + if args.solver: + mm.lap.default_solver = args.solver + + gtfiles = glob.glob(os.path.join(args.groundtruths, '*')) + tsfiles = glob.glob(os.path.join(args.tests, '*')) + + logging.info('Found {} groundtruths and {} test files.'.format(len(gtfiles), len(tsfiles))) + logging.info('Available LAP solvers {}'.format(mm.lap.available_solvers)) + logging.info('Default LAP solver \'{}\''.format(mm.lap.default_solver)) + logging.info('Loading files.') + + gt = OrderedDict([(os.path.splitext(Path(f).parts[-1])[0], mm.io.loadtxt(f, fmt=args.gtfmt)) for f in gtfiles]) + ts = OrderedDict([(os.path.splitext(Path(f).parts[-1])[0], mm.io.loadtxt(f, fmt=args.tsfmt)) for f in tsfiles]) + + mh = mm.metrics.create() + accs, names = compare_dataframes(gt, ts) + + logging.info('Running metrics') + + summary = mh.compute_many(accs, names=names, metrics=mm.metrics.motchallenge_metrics, generate_overall=True) + print(mm.io.render_summary(summary, formatters=mh.formatters, namemap=mm.io.motchallenge_metric_names)) + logging.info('Completed') \ No newline at end of file diff --git a/motmetrics/data/iotest/detrac.mat b/motmetrics/data/iotest/detrac.mat new file mode 100644 index 0000000000000000000000000000000000000000..fc5c2a6668a0a277f353e306c1867f91cdf0b914 GIT binary patch literal 2752 zcmeZu4DoSvQZUssQ1EpO(M`+DNmU5QNi0drFUqx2D9A6)tk6+#E=o--Nlj76&$Chp z2h#q@B?`s{3WnxZMy6HbS2T2W$ds$XetF;oO({|BIDF#AC? z2*B)T0kZXg7~}>J-~{3WAW0Bd(CGlCW1zGGl)lpe5eG|-5~Cq78UmvsFd71bDFi^} z{sLg34lDP$p^n21pPd0xH`D;R{+%+huY0KZ735x!e=UF*7Qg8E5R}nCc@LK1Kx|MukrCKZWCqFr E0G?VhMgRZ+ literal 0 HcmV?d00001 diff --git a/motmetrics/data/iotest/detrac.xml b/motmetrics/data/iotest/detrac.xml new file mode 100644 index 00000000..f649e8f6 --- /dev/null +++ b/motmetrics/data/iotest/detrac.xml @@ -0,0 +1,41 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/motmetrics/io.py b/motmetrics/io.py index 3427a134..69041a33 100644 --- a/motmetrics/io.py +++ b/motmetrics/io.py @@ -12,6 +12,8 @@ import pandas as pd import numpy as np import io +import scipy.io +import xmltodict class Format(Enum): """Enumerates supported file formats.""" @@ -27,6 +29,17 @@ class Format(Enum): https://github.com/cvondrick/vatic """ + DETRAC_MAT = 'detrac-mat' + """Wen, Longyin et al. "UA-DETRAC: A New Benchmark and Protocol for Multi-Object Detection and Tracking." arXiv preprint arXiv:arXiv:1511.04136 (2016). + http://detrac-db.rit.albany.edu/download + """ + + DETRAC_XML = 'detrac-xml' + """Wen, Longyin et al. "UA-DETRAC: A New Benchmark and Protocol for Multi-Object Detection and Tracking." arXiv preprint arXiv:arXiv:1511.04136 (2016). + http://detrac-db.rit.albany.edu/download + """ + + def load_motchallenge(fname, **kwargs): """Load MOT challenge data. @@ -162,6 +175,126 @@ def load_vatictxt(fname, **kwargs): return df +def load_detrac_mat(fname, **kwargs): + """Loads UA-DETRAC annotations data from mat files + Competition Site: http://detrac-db.rit.albany.edu/download + + File contains a nested structure of 2d arrays for indexed by frame id + and Object ID. Separate arrays for top, left, width and height are given. + + Params + ------ + fname : str + Filename to load data from + + Kwargs + ------ + Currently none of these arguments used. + + Returns + ------ + df : pandas.DataFrame + The returned dataframe has the following columns + 'X', 'Y', 'Width', 'Height', 'Confidence', 'ClassId', 'Visibility' + The dataframe is indexed by ('FrameId', 'Id') + """ + + matData = scipy.io.loadmat(fname) + + frameList = matData['gtInfo'][0][0][4][0] + leftArray = matData['gtInfo'][0][0][0] + topArray = matData['gtInfo'][0][0][1] + widthArray = matData['gtInfo'][0][0][3] + heightArray = matData['gtInfo'][0][0][2] + + parsedGT = [] + for f in frameList: + ids = [i+1 for i,v in enumerate(leftArray[f-1]) if v>0] + for i in ids: + row = [] + row.append(f) + row.append(i) + row.append(leftArray[f-1,i-1] - widthArray[f-1,i-1] / 2) + row.append(topArray[f-1,i-1] - heightArray[f-1,i-1]) + row.append(widthArray[f-1,i-1]) + row.append(heightArray[f-1,i-1]) + row.append(1) + row.append(-1) + row.append(-1) + row.append(-1) + parsedGT.append(row) + + df = pd.DataFrame(parsedGT, + columns=['FrameId', 'Id', 'X', 'Y', 'Width', 'Height', 'Confidence', 'ClassId', 'Visibility', 'unused']) + df.set_index(['FrameId', 'Id'],inplace=True) + + # Account for matlab convention. + df[['X', 'Y']] -= (1, 1) + + # Removed trailing column + del df['unused'] + + return df + +def load_detrac_xml(fname, **kwargs): + """Loads UA-DETRAC annotations data from xml files + Competition Site: http://detrac-db.rit.albany.edu/download + + Params + ------ + fname : str + Filename to load data from + + Kwargs + ------ + Currently none of these arguments used. + + Returns + ------ + df : pandas.DataFrame + The returned dataframe has the following columns + 'X', 'Y', 'Width', 'Height', 'Confidence', 'ClassId', 'Visibility' + The dataframe is indexed by ('FrameId', 'Id') + """ + + with open(fname) as fd: + doc = xmltodict.parse(fd.read()) + frameList = doc['sequence']['frame'] + + parsedGT = [] + for f in frameList: + fid = int(f['@num']) + targetList = f['target_list']['target'] + if type(targetList) != list: + targetList = [targetList] + + for t in targetList: + row = [] + row.append(fid) + row.append(int(t['@id'])) + row.append(float(t['box']['@left'])) + row.append(float(t['box']['@top'])) + row.append(float(t['box']['@width'])) + row.append(float(t['box']['@height'])) + row.append(1) + row.append(-1) + row.append(-1) + row.append(-1) + parsedGT.append(row) + + df = pd.DataFrame(parsedGT, + columns=['FrameId', 'Id', 'X', 'Y', 'Width', 'Height', 'Confidence', 'ClassId', 'Visibility', 'unused']) + df.set_index(['FrameId', 'Id'],inplace=True) + + # Account for matlab convention. + df[['X', 'Y']] -= (1, 1) + + # Removed trailing column + del df['unused'] + + return df + + def loadtxt(fname, fmt=Format.MOT15_2D, **kwargs): """Load data from any known format.""" fmt = Format(fmt) @@ -169,7 +302,9 @@ def loadtxt(fname, fmt=Format.MOT15_2D, **kwargs): switcher = { Format.MOT16: load_motchallenge, Format.MOT15_2D: load_motchallenge, - Format.VATIC_TXT: load_vatictxt + Format.VATIC_TXT: load_vatictxt, + Format.DETRAC_MAT: load_detrac_mat, + Format.DETRAC_XML: load_detrac_xml } func = switcher.get(fmt) return func(fname, **kwargs) diff --git a/motmetrics/tests/test_io.py b/motmetrics/tests/test_io.py index f206b012..19ccf29d 100644 --- a/motmetrics/tests/test_io.py +++ b/motmetrics/tests/test_io.py @@ -28,4 +28,28 @@ def test_load_motchallenge(): (2,4,199,205,55,137,1,-1,-1), ]) - assert (df.reset_index().values == expected.values).all() \ No newline at end of file + assert (df.reset_index().values == expected.values).all() + +def test_load_detrac_mat(): + df = mm.io.loadtxt(os.path.join(DATA_DIR, 'iotest/detrac.mat'), fmt=mm.io.Format.DETRAC_MAT) + + expected = pd.DataFrame([ + ( 1., 1., 745., 356., 148., 115., 1., -1., -1.), + ( 2., 1., 738., 350., 145., 111., 1., -1., -1.), + ( 3., 1., 732., 343., 142., 107., 1., -1., -1.), + ( 4., 1., 725., 336., 139., 104., 1., -1., -1.) + ]) + + assert (df.reset_index().values == expected.values).all() + +def test_load_detrac_xml(): + df = mm.io.loadtxt(os.path.join(DATA_DIR, 'iotest/detrac.xml'), fmt=mm.io.Format.DETRAC_XML) + + expected = pd.DataFrame([ + ( 1. , 1. , 744.6 , 356.33, 148.2 , 115.14, 1. , -1. , -1. ), + ( 2. , 1. , 738.2 , 349.51, 145.21, 111.29, 1. , -1. , -1. ), + ( 3. , 1. , 731.8 , 342.68, 142.23, 107.45, 1. , -1. , -1. ), + ( 4. , 1. , 725.4 , 335.85, 139.24, 103.62, 1. , -1. , -1. ) + ]) + + assert (df.reset_index().values == expected.values).all() diff --git a/requirements.txt b/requirements.txt index 1d497e2f..d05b63c9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ numpy>=1.12.1 pandas>=0.23.1 scipy>=0.19.0 +xmltodict>=0.12.0