@article{bb0d9a01f03042f1b9ad8d65c39c27e8,
title = "Crowd-sourced and expert video assessment in minimally invasive esophagectomy",
abstract = "Background: Video-based assessment by experts may structurally measure surgical performance using procedure-specific competency assessment tools (CATs). A CAT for minimally invasive esophagectomy (MIE-CAT) was developed and validated previously. However, surgeon{\textquoteright}s time is scarce and video assessment is time-consuming and labor intensive. This study investigated non-procedure-specific assessment of MIE video clips by MIE experts and crowdsourcing, collective surgical performance evaluation by anonymous and untrained laypeople, to assist procedure-specific expert review. Methods: Two surgical performance scoring frameworks were used to assess eight MIE videos. First, global performance was assessed with the non-procedure-specific Global Operative Assessment of Laparoscopic Skills (GOALS) of 64 procedural phase-based video clips < 10 min. Each clip was assessed by two MIE experts and > 30 crowd workers. Second, the same experts assessed procedure-specific performance with the MIE-CAT of the corresponding full-length video. Reliability and convergent validity of GOALS for MIE were investigated using hypothesis testing with correlations (experience, blood loss, operative time, and MIE-CAT). Results: Less than 75% of hypothesized correlations between GOALS scores and experience of the surgical team (r < 0.3), blood loss (r = − 0.82 to 0.02), operative time (r = − 0.42 to 0.07), and the MIE-CAT scores (r = − 0.04 to 0.76) were met for both crowd workers and experts. Interestingly, experts{\textquoteright} GOALS and MIE-CAT scores correlated strongly (r = 0.40 to 0.79), while crowd workers{\textquoteright} GOALS and experts{\textquoteright} MIE-CAT scores correlations were weak (r = − 0.04 to 0.49). Expert and crowd worker GOALS scores correlated poorly (ICC ≤ 0.42). Conclusion: GOALS assessments by crowd workers lacked convergent validity and showed poor reliability. It is likely that MIE is technically too difficult to assess for laypeople. Convergent validity of GOALS assessments by experts could also not be established. GOALS might not be comprehensive enough to assess detailed MIE performance. However, expert{\textquoteright}s GOALS and MIE-CAT scores strongly correlated indicating video clip (instead of full-length video) assessments could be useful to shorten assessment time. Graphical abstract: [Figure not available: see fulltext.]",
keywords = "Competency assessment tool, Crowdsourcing, Esophagectomy, GOALS, Surgical performance assessment, Video assessment",
author = "Ketel, {Mirte H. M.} and Klarenbeek, {Bastiaan R.} and Yassin Eddahchouri and Cuesta, {Miguel A.} and {van Daele}, Elke and Gutschow, {Christian A.} and H{\"o}lscher, {Arnulf H.} and Michal Hubka and Luyer, {Misha D. P.} and Merritt, {Robert E.} and Nieuwenhuijzen, {Grard A. P.} and Yaxing Shen and Abma, {Inger L.} and Camiel Rosman and {van Workum}, Frans",
note = "Funding Information: This study was funded via the investigator-initiated study proposal rewarded by Ethicon Endo-Surgery by Johnson & Johnson Medical Limited. The study was performed independently, and Ethicon had no role in the design and conduct of the study; collection, management, analysis, and interpretation of the data; preparation, review, or approval of the manuscript; and decision to submit to manuscript. Funding Information: Mirte Ketel, Bastiaan Klarenbeek, Camiel Rosman, Yassin Eddahchouri, and Frans van Workum declare funding from Ethicon for the submitted work. However, Ethicon as sponsor had no role in the design and conduct of the study: collection, management, analysis, and interpretation of the data; preparation, review, or approval of the manuscript; and decision to submit the manuscript for publication. Both Camiel Rosman and Bastiaan Klarenbeek report grants from Medtronic and Bastiaan Klarenbeek reports grants from ZonMW, Stichting Bergh in het Zadel, BeNeFit, Stichting Sasha Swartouw, and Radboud Oncology Fund outside of the submitted work. In addition, C-SATS donated the use of their product for the completion of this study. Elke van Daele reports a study grant {\textquoteleft}Kom op tegen kanker{\textquoteright} outside of the submitted work. Christian Gutschow received speaker honoraria: BD/Bard Medical, Johnson & Johnson, B. Braun/Aesculap, Micro-Tech Europe GmbH, and Bristol-Myers Squibb; Consulting fees: BD/Bard Medical and Medtronic; and Research grant: B. Braun/Aesculap, all outside of the submitted work. Misha Luyer received consulting fees from Medtronic and Galvani, outside of the submitted work. Robert Merritt received consulting fees from Astra Zeneca as advisory board and Honoria from Intuitive Surgical for lectures, all outside of the submitted work. Grard Nieuwenhuijzen received consulting fees from Medtronic, outside of the submitted work. Inger Abma, Miguel Cuesta, Michal Hubka, Arnulf H{\"o}lscher, and Yaxing Shen have no conflicts of interest or financial ties to disclose. Publisher Copyright: {\textcopyright} 2023, The Author(s).",
year = "2023",
month = oct,
doi = "https://doi.org/10.1007/s00464-023-10297-2",
language = "English",
volume = "37",
pages = "7819--7828",
journal = "Surgical endoscopy",
issn = "0930-2794",
publisher = "Springer New York",
number = "10",
}