@Article{Deriu2020Survey,
author={Deriu, Jan
and Rodrigo, Alvaro
and Otegi, Arantxa
and Echegoyen, Guillermo
and Rosset, Sophie
and Agirre, Eneko
and Cieliebak, Mark},
title={Survey on evaluation methods for dialogue systems},
journal={Artificial Intelligence Review},
year={2020},
month={Jun},
day={25},
abstract={In this paper, we survey the methods and concepts developed for the evaluation of dialogue systems. Evaluation, in and of itself, is a crucial part during the development process. Often, dialogue systems are evaluated by means of human evaluations and questionnaires. However, this tends to be very cost- and time-intensive. Thus, much work has been put into finding methods which allow a reduction in involvement of human labour. In this survey, we present the main concepts and methods. For this, we differentiate between the various classes of dialogue systems (task-oriented, conversational, and question-answering dialogue systems). We cover each class by introducing the main technologies developed for the dialogue systems and then present the evaluation methods regarding that class.},
issn={1573-7462},
doi={10.1007/s10462-020-09866-x},
url={https://doi.org/10.1007/s10462-020-09866-x}
}