% IMPORTANT: The following is UTF-8 encoded.  This means that in the presence
% of non-ASCII characters, it will not work with BibTeX 0.99 or older.
% Instead, you should use an up-to-date BibTeX implementation like “bibtex8” or
% “biber”.

@ARTICLE{Heim:141663,
      author       = {E. Heim$^*$ and A. Seitel$^*$ and J. Andrulis and F.
                      Isensee$^*$ and C. Stock$^*$ and T. Ross$^*$ and L.
                      Maier-Hein$^*$},
      title        = {{C}lickstream {A}nalysis for {C}rowd-{B}ased {O}bject
                      {S}egmentation with {C}onfidence.},
      journal      = {IEEE transactions on pattern analysis and machine
                      intelligence},
      volume       = {40},
      number       = {12},
      issn         = {0162-8828},
      address      = {New York, NY},
      publisher    = {IEEE},
      reportid     = {DKFZ-2018-01934},
      pages        = {2814 - 2826},
      year         = {2018},
      note         = {IEEE Transactions on Pattern Analysis and Machine
                      Intelligence (IEEE Trans. Pattern Anal. Mach. Intell.) =
                      2160-92921939-3539 (import from CrossRef, PubMed, )},
      abstract     = {With the rapidly increasing interest in machine learning
                      based solutions for automatic image annotation, the
                      availability of reference annotations for algorithm training
                      is one of the major bottlenecks in the field. Crowdsourcing
                      has evolved as a valuable option for low-cost and
                      large-scale data annotation; however, quality control
                      remains a major issue which needs to be addressed. To our
                      knowledge, we are the first to analyze the annotation
                      process to improve crowd-sourced image segmentation. Our
                      method involves training a regressor to estimate the quality
                      of a segmentation from the annotator's clickstream data. The
                      quality estimation can be used to identify spam and weight
                      individual annotations by their (estimated) quality when
                      merging multiple segmentations of one image. Using a total
                      of 29,000 crowd annotations performed on publicly available
                      data of different object classes, we show that (1) our
                      method is highly accurate in estimating the segmentation
                      quality based on clickstream data, (2) outperforms
                      state-of-the-art methods for merging multiple annotations.
                      As the regressor does not need to be trained on the object
                      class that it is applied to it can be regarded as a low-cost
                      option for quality control and confidence analysis in the
                      context of crowd-based image annotation.},
      cin          = {E130 / E132 / C070},
      ddc          = {004},
      cid          = {I:(DE-He78)E130-20160331 / I:(DE-He78)E132-20160331 /
                      I:(DE-He78)C070-20160331},
      pnm          = {315 - Imaging and radiooncology (POF3-315)},
      pid          = {G:(DE-HGF)POF3-315},
      typ          = {PUB:(DE-HGF)16},
      pubmed       = {pmid:29989983},
      doi          = {10.1109/TPAMI.2017.2777967},
      url          = {https://inrepo02.dkfz.de/record/141663},
}