000288083 001__ 288083
000288083 005__ 20250326103243.0
000288083 0247_ $$2doi$$a10.1038/s41592-023-02151-z
000288083 0247_ $$2pmid$$apmid:38347141
000288083 0247_ $$2ISSN$$a1548-7091
000288083 0247_ $$2ISSN$$a1548-7105
000288083 0247_ $$2altmetric$$aaltmetric:159461769
000288083 037__ $$aDKFZ-2024-00337
000288083 041__ $$aEnglish
000288083 082__ $$a610
000288083 1001_ $$0P:(DE-He78)26a1176cd8450660333a012075050072$$aMaier-Hein, Lena$$b0$$eFirst author
000288083 245__ $$aMetrics reloaded: recommendations for image analysis validation.
000288083 260__ $$aLondon [u.a.]$$bNature Publishing Group$$c2024
000288083 3367_ $$2DRIVER$$aarticle
000288083 3367_ $$2DataCite$$aOutput Types/Journal article
000288083 3367_ $$0PUB:(DE-HGF)16$$2PUB:(DE-HGF)$$aJournal Article$$bjournal$$mjournal$$s1710237197_32557$$xReview Article
000288083 3367_ $$2BibTeX$$aARTICLE
000288083 3367_ $$2ORCID$$aJOURNAL_ARTICLE
000288083 3367_ $$00$$2EndNote$$aJournal Article
000288083 500__ $$a#EA:E130#LA:E290#
000288083 520__ $$aIncreasing evidence shows that flaws in machine learning (ML) algorithm validation are an underestimated global problem. In biomedical image analysis, chosen performance metrics often do not reflect the domain interest, and thus fail to adequately measure scientific progress and hinder translation of ML techniques into practice. To overcome this, we created Metrics Reloaded, a comprehensive framework guiding researchers in the problem-aware selection of metrics. Developed by a large international consortium in a multistage Delphi process, it is based on the novel concept of a problem fingerprint-a structured representation of the given problem that captures all aspects that are relevant for metric selection, from the domain interest to the properties of the target structure(s), dataset and algorithm output. On the basis of the problem fingerprint, users are guided through the process of choosing and applying appropriate validation metrics while being made aware of potential pitfalls. Metrics Reloaded targets image analysis problems that can be interpreted as classification tasks at image, object or pixel level, namely image-level classification, object detection, semantic segmentation and instance segmentation tasks. To improve the user experience, we implemented the framework in the Metrics Reloaded online tool. Following the convergence of ML methodology across application domains, Metrics Reloaded fosters the convergence of validation methodology. Its applicability is demonstrated for various biomedical use cases.
000288083 536__ $$0G:(DE-HGF)POF4-315$$a315 - Bildgebung und Radioonkologie (POF4-315)$$cPOF4-315$$fPOF IV$$x0
000288083 588__ $$aDataset connected to CrossRef, PubMed, , Journals: inrepo02.dkfz.de
000288083 7001_ $$0P:(DE-He78)97e904f47dab556a77c0149cd0002591$$aReinke, Annika$$b1$$eFirst author
000288083 7001_ $$0P:(DE-He78)77a2a5b07dcbd46277a18a32372ea154$$aGodau, Patrick$$b2
000288083 7001_ $$0P:(DE-He78)26651d9aa10255ad4f35610a56aa91e8$$aTizabi, Minu D$$b3
000288083 7001_ $$0P:(DE-He78)fdeeec93551e8f3bad68db88a1130c5d$$aBüttner, Florian$$b4$$udkfz
000288083 7001_ $$0P:(DE-He78)8da2eca0bc6341c8681c317fe2b8e27b$$aChristodoulou, Evangelia$$b5
000288083 7001_ $$00000-0002-4897-9356$$aGlocker, Ben$$b6
000288083 7001_ $$0P:(DE-He78)7ea9af59d03ec7deb982a0e0562358fa$$aIsensee, Fabian$$b7
000288083 7001_ $$aKleesiek, Jens$$b8
000288083 7001_ $$00000-0001-7902-589X$$aKozubek, Michal$$b9
000288083 7001_ $$aReyes, Mauricio$$b10
000288083 7001_ $$00000-0002-3153-2064$$aRiegler, Michael A$$b11
000288083 7001_ $$0P:(DE-He78)1042737c83ba70ec508bdd99f0096864$$aWiesenfarth, Manuel$$b12
000288083 7001_ $$0P:(DE-He78)64296d0922a2da68074f5de2ccf74487$$aKavur, A Emre$$b13
000288083 7001_ $$aSudre, Carole H$$b14
000288083 7001_ $$0P:(DE-He78)6ae68adec0915a4ae2d58aff814aaceb$$aBaumgartner, Michael$$b15
000288083 7001_ $$0P:(DE-He78)c9d6245b17f0ab26eeed345cb00d3359$$aEisenmann, Matthias$$b16
000288083 7001_ $$0P:(DE-He78)98dd16a42af0888dd129e1cd2ee2766a$$aHeckmann-Nötzel, Doreen$$b17
000288083 7001_ $$0P:(DE-He78)683f71d452c83a7f5ec62969c6012466$$aRädsch, Tim$$b18
000288083 7001_ $$00000-0001-5213-6012$$aAcion, Laura$$b19
000288083 7001_ $$00000-0002-3005-4523$$aAntonelli, Michela$$b20
000288083 7001_ $$00000-0001-8870-3007$$aArbel, Tal$$b21
000288083 7001_ $$00000-0001-8734-6482$$aBakas, Spyridon$$b22
000288083 7001_ $$00000-0002-9125-8300$$aBenis, Arriel$$b23
000288083 7001_ $$aBlaschko, Matthew B$$b24
000288083 7001_ $$00000-0003-1284-2558$$aCardoso, M Jorge$$b25
000288083 7001_ $$aCheplygina, Veronika$$b26
000288083 7001_ $$00000-0001-9640-9318$$aCimini, Beth A$$b27
000288083 7001_ $$00000-0002-2772-2316$$aCollins, Gary S$$b28
000288083 7001_ $$aFarahani, Keyvan$$b29
000288083 7001_ $$aFerrer, Luciana$$b30
000288083 7001_ $$aGaldran, Adrian$$b31
000288083 7001_ $$avan Ginneken, Bram$$b32
000288083 7001_ $$aHaase, Robert$$b33
000288083 7001_ $$00000-0003-4725-3104$$aHashimoto, Daniel A$$b34
000288083 7001_ $$00000-0002-4517-1562$$aHoffman, Michael M$$b35
000288083 7001_ $$aHuisman, Merel$$b36
000288083 7001_ $$00000-0002-7415-071X$$aJannin, Pierre$$b37
000288083 7001_ $$00000-0002-6654-7434$$aKahn, Charles E$$b38
000288083 7001_ $$aKainmueller, Dagmar$$b39
000288083 7001_ $$aKainz, Bernhard$$b40
000288083 7001_ $$00000-0002-1930-3410$$aKarargyris, Alexandros$$b41
000288083 7001_ $$aKarthikesalingam, Alan$$b42
000288083 7001_ $$aKofler, Florian$$b43
000288083 7001_ $$0P:(DE-He78)bb6a7a70f976eb8df1769944bf913596$$aKopp-Schneider, Annette$$b44
000288083 7001_ $$00000-0003-1334-6388$$aKreshuk, Anna$$b45
000288083 7001_ $$aKurc, Tahsin$$b46
000288083 7001_ $$00000-0001-5733-2127$$aLandman, Bennett A$$b47
000288083 7001_ $$00000-0003-1554-1291$$aLitjens, Geert$$b48
000288083 7001_ $$aMadani, Amin$$b49
000288083 7001_ $$0P:(DE-He78)33c74005e1ce56f7025c4f6be15321b3$$aMaier-Hein, Klaus$$b50
000288083 7001_ $$00000-0003-1375-5501$$aMartel, Anne L$$b51
000288083 7001_ $$00000-0002-5984-238X$$aMattson, Peter$$b52
000288083 7001_ $$00000-0001-8015-8358$$aMeijering, Erik$$b53
000288083 7001_ $$00000-0003-4136-5690$$aMenze, Bjoern$$b54
000288083 7001_ $$aMoons, Karel G M$$b55
000288083 7001_ $$00000-0001-6800-9878$$aMüller, Henning$$b56
000288083 7001_ $$00009-0006-8087-6089$$aNichyporuk, Brennan$$b57
000288083 7001_ $$aNickel, Felix$$b58
000288083 7001_ $$aPetersen, Jens$$b59
000288083 7001_ $$00000-0001-6760-1271$$aRajpoot, Nasir$$b60
000288083 7001_ $$00000-0003-0241-9334$$aRieke, Nicola$$b61
000288083 7001_ $$aSaez-Rodriguez, Julio$$b62
000288083 7001_ $$aSánchez, Clara I$$b63
000288083 7001_ $$aShetty, Shravya$$b64
000288083 7001_ $$avan Smeden, Maarten$$b65
000288083 7001_ $$00000-0001-8081-7376$$aSummers, Ronald M$$b66
000288083 7001_ $$00000-0002-7604-9041$$aTaha, Abdel A$$b67
000288083 7001_ $$00000-0002-7852-4141$$aTiulpin, Aleksei$$b68
000288083 7001_ $$aTsaftaris, Sotirios A$$b69
000288083 7001_ $$aVan Calster, Ben$$b70
000288083 7001_ $$00000-0003-1076-5122$$aVaroquaux, Gaël$$b71
000288083 7001_ $$0P:(DE-He78)04a0b5a49db132d8f00cee326cb743ca$$aJäger, Paul$$b72$$eLast author
000288083 773__ $$0PERI:(DE-600)2163081-1$$a10.1038/s41592-023-02151-z$$gVol. 21, no. 2, p. 195 - 212$$n2$$p195 - 212$$tNature methods$$v21$$x1548-7091$$y2024
000288083 8564_ $$uhttps://inrepo02.dkfz.de/record/288083/files/s41592-023-02151-z.pdf
000288083 8564_ $$uhttps://inrepo02.dkfz.de/record/288083/files/s41592-023-02151-z.pdf?subformat=pdfa$$xpdfa
000288083 909CO $$ooai:inrepo02.dkfz.de:288083$$pVDB
000288083 9101_ $$0I:(DE-588b)2036810-0$$6P:(DE-He78)26a1176cd8450660333a012075050072$$aDeutsches Krebsforschungszentrum$$b0$$kDKFZ
000288083 9101_ $$0I:(DE-588b)2036810-0$$6P:(DE-He78)97e904f47dab556a77c0149cd0002591$$aDeutsches Krebsforschungszentrum$$b1$$kDKFZ
000288083 9101_ $$0I:(DE-588b)2036810-0$$6P:(DE-He78)77a2a5b07dcbd46277a18a32372ea154$$aDeutsches Krebsforschungszentrum$$b2$$kDKFZ
000288083 9101_ $$0I:(DE-588b)2036810-0$$6P:(DE-He78)26651d9aa10255ad4f35610a56aa91e8$$aDeutsches Krebsforschungszentrum$$b3$$kDKFZ
000288083 9101_ $$0I:(DE-588b)2036810-0$$6P:(DE-He78)fdeeec93551e8f3bad68db88a1130c5d$$aDeutsches Krebsforschungszentrum$$b4$$kDKFZ
000288083 9101_ $$0I:(DE-588b)2036810-0$$6P:(DE-He78)8da2eca0bc6341c8681c317fe2b8e27b$$aDeutsches Krebsforschungszentrum$$b5$$kDKFZ
000288083 9101_ $$0I:(DE-588b)2036810-0$$6P:(DE-He78)7ea9af59d03ec7deb982a0e0562358fa$$aDeutsches Krebsforschungszentrum$$b7$$kDKFZ
000288083 9101_ $$0I:(DE-588b)2036810-0$$6P:(DE-He78)1042737c83ba70ec508bdd99f0096864$$aDeutsches Krebsforschungszentrum$$b12$$kDKFZ
000288083 9101_ $$0I:(DE-588b)2036810-0$$6P:(DE-He78)64296d0922a2da68074f5de2ccf74487$$aDeutsches Krebsforschungszentrum$$b13$$kDKFZ
000288083 9101_ $$0I:(DE-588b)2036810-0$$6P:(DE-He78)6ae68adec0915a4ae2d58aff814aaceb$$aDeutsches Krebsforschungszentrum$$b15$$kDKFZ
000288083 9101_ $$0I:(DE-588b)2036810-0$$6P:(DE-He78)c9d6245b17f0ab26eeed345cb00d3359$$aDeutsches Krebsforschungszentrum$$b16$$kDKFZ
000288083 9101_ $$0I:(DE-588b)2036810-0$$6P:(DE-He78)98dd16a42af0888dd129e1cd2ee2766a$$aDeutsches Krebsforschungszentrum$$b17$$kDKFZ
000288083 9101_ $$0I:(DE-588b)2036810-0$$6P:(DE-He78)683f71d452c83a7f5ec62969c6012466$$aDeutsches Krebsforschungszentrum$$b18$$kDKFZ
000288083 9101_ $$0I:(DE-588b)2036810-0$$6P:(DE-He78)bb6a7a70f976eb8df1769944bf913596$$aDeutsches Krebsforschungszentrum$$b44$$kDKFZ
000288083 9101_ $$0I:(DE-588b)2036810-0$$6P:(DE-He78)33c74005e1ce56f7025c4f6be15321b3$$aDeutsches Krebsforschungszentrum$$b50$$kDKFZ
000288083 9101_ $$0I:(DE-588b)2036810-0$$6P:(DE-He78)04a0b5a49db132d8f00cee326cb743ca$$aDeutsches Krebsforschungszentrum$$b72$$kDKFZ
000288083 9131_ $$0G:(DE-HGF)POF4-315$$1G:(DE-HGF)POF4-310$$2G:(DE-HGF)POF4-300$$3G:(DE-HGF)POF4$$4G:(DE-HGF)POF$$aDE-HGF$$bGesundheit$$lKrebsforschung$$vBildgebung und Radioonkologie$$x0
000288083 9141_ $$y2024
000288083 915__ $$0StatID:(DE-HGF)0420$$2StatID$$aNationallizenz$$d2023-08-29$$wger
000288083 915__ $$0StatID:(DE-HGF)3003$$2StatID$$aDEAL Nature$$d2023-08-29$$wger
000288083 915__ $$0StatID:(DE-HGF)0200$$2StatID$$aDBCoverage$$bSCOPUS$$d2023-08-29
000288083 915__ $$0StatID:(DE-HGF)0300$$2StatID$$aDBCoverage$$bMedline$$d2023-08-29
000288083 915__ $$0StatID:(DE-HGF)0199$$2StatID$$aDBCoverage$$bClarivate Analytics Master Journal List$$d2023-08-29
000288083 915__ $$0StatID:(DE-HGF)1050$$2StatID$$aDBCoverage$$bBIOSIS Previews$$d2023-08-29
000288083 915__ $$0StatID:(DE-HGF)0113$$2StatID$$aWoS$$bScience Citation Index Expanded$$d2023-08-29
000288083 915__ $$0StatID:(DE-HGF)0150$$2StatID$$aDBCoverage$$bWeb of Science Core Collection$$d2023-08-29
000288083 915__ $$0StatID:(DE-HGF)1030$$2StatID$$aDBCoverage$$bCurrent Contents - Life Sciences$$d2023-08-29
000288083 915__ $$0StatID:(DE-HGF)1190$$2StatID$$aDBCoverage$$bBiological Abstracts$$d2023-08-29
000288083 915__ $$0StatID:(DE-HGF)0160$$2StatID$$aDBCoverage$$bEssential Science Indicators$$d2023-08-29
000288083 915__ $$0StatID:(DE-HGF)0100$$2StatID$$aJCR$$bNAT METHODS : 2022$$d2023-08-29
000288083 915__ $$0StatID:(DE-HGF)0600$$2StatID$$aDBCoverage$$bEbsco Academic Search$$d2023-08-29
000288083 915__ $$0StatID:(DE-HGF)0030$$2StatID$$aPeer Review$$bASC$$d2023-08-29
000288083 915__ $$0StatID:(DE-HGF)9940$$2StatID$$aIF >= 40$$bNAT METHODS : 2022$$d2023-08-29
000288083 9202_ $$0I:(DE-He78)E290-20160331$$kE290$$lNWG Interaktives maschinelles Lernen$$x0
000288083 9201_ $$0I:(DE-He78)E130-20160331$$kE130$$lE130 Intelligente Medizinische Systeme$$x0
000288083 9201_ $$0I:(DE-He78)FM01-20160331$$kFM01$$lDKTK Koordinierungsstelle Frankfurt$$x1
000288083 9201_ $$0I:(DE-He78)E230-20160331$$kE230$$lE230 Medizinische Bildverarbeitung$$x2
000288083 9201_ $$0I:(DE-He78)C060-20160331$$kC060$$lC060 Biostatistik$$x3
000288083 9201_ $$0I:(DE-He78)E290-20160331$$kE290$$lNWG Interaktives maschinelles Lernen$$x4
000288083 9200_ $$0I:(DE-He78)E130-20160331$$kE130$$lE130 Intelligente Medizinische Systeme$$x0
000288083 980__ $$ajournal
000288083 980__ $$aVDB
000288083 980__ $$aI:(DE-He78)E130-20160331
000288083 980__ $$aI:(DE-He78)FM01-20160331
000288083 980__ $$aI:(DE-He78)E230-20160331
000288083 980__ $$aI:(DE-He78)C060-20160331
000288083 980__ $$aI:(DE-He78)E290-20160331
000288083 980__ $$aUNRESTRICTED