Project homeWiki source code of j. SFT1 evaluation templates
Version 26.3 by Rosa Van Tuijn on 2026/05/05 08:30
Show last authors
| author | version | line-number | content |
|---|---|---|---|
| 1 | = General = | ||
| 2 | |||
| 3 | [[image:1777962330814-818.png]] | ||
| 4 | |||
| 5 | |||
| 6 | |(% colspan="3" %)[[image:1776259393121-951.png]] | ||
| 7 | |**Baseline phases**|**Who fills it**|**Main purpose** | ||
| 8 | |//Briefing//|-|- | ||
| 9 | |//Estimate building integrity//|-|- | ||
| 10 | |//Shoring//|Particpants|(make estimation about shoring time) | ||
| 11 | |//Before sending in FRs (DM)//|Participant|Ground truth/ baseline SA | ||
| 12 | |//Send in FRs, identify voids, DM//|Evaluation leader|Objective performance, safety, timing, decision making | ||
| 13 | |//FRs exit building//|Participant|Subjective SA, trust, workload, decision making | ||
| 14 | |//Hotwash//| | | ||
| 15 | |||
| 16 | |(% colspan="3" style="width:414px" %)[[image:1776259356197-904.png]] | ||
| 17 | |(% style="width:414px" %)**SYNERGISE phases**|(% style="width:245px" %)**Who fills it**|(% style="width:690px" %)**Main purpose** | ||
| 18 | |(% style="width:414px" %)//Briefing//|(% style="width:245px" %)-|(% style="width:690px" %)- | ||
| 19 | |(% style="width:414px" %)//Estimate building integrity//|(% style="width:245px" %)-|(% style="width:690px" %)- | ||
| 20 | |(% style="width:414px" %)//Send in robots, identify voids with robots//|(% style="width:245px" %)-|(% style="width:690px" %)- | ||
| 21 | |(% style="width:414px" %)//Decision Making (DM)//|(% style="width:245px" %)Participants (Squad/ team leader)|(% style="width:690px" %)Ground truth/ baseline SA | ||
| 22 | |(% style="width:414px" %)//Send in FRs ?//|(% style="width:245px" %)Evaluation leader|(% style="width:690px" %)Objective performance, safety, timing, decision making | ||
| 23 | |(% style="width:414px" %)//After test//|(% style="width:245px" %)Paricipant|(% style="width:690px" %)Subjective SA, trust, workload, decision making | ||
| 24 | |(% style="width:414px" %)//Hotwash//|(% style="width:245px" %) |(% style="width:690px" %) | ||
| 25 | |||
| 26 | == List of ground truths needed for comparison: == | ||
| 27 | |||
| 28 | * Health ranges | ||
| 29 | * hazards/ victims in buildings + Safest path within buildings : Safety observations + path observations >> have a premade map with everything and note down the route. Have an expert rate it after the fact. | ||
| 30 | * Decision quality: have expert (or team leaders) rate the path of the FRs inside | ||
| 31 | * State of automation (tech question) | ||
| 32 | |||
| 33 | = Templates per role = | ||
| 34 | |||
| 35 | (% class="table-bordered" %) | ||
| 36 | |(% colspan="2" %)((( | ||
| 37 | == Template: Evaluation leader (Field) == | ||
| 38 | ))) | ||
| 39 | |(% colspan="2" %)((( | ||
| 40 | **General: ** | ||
| 41 | |||
| 42 | //Trial type/ Building/ scenario ID//: ................................. | //Date/ Time//: ....................................... | //Condition (encircle)//: Baseline / SYNERGISE-tech supported | ||
| 43 | ))) | ||
| 44 | |(% style="width:296px" %)((( | ||
| 45 | System & mission readiness | ||
| 46 | |||
| 47 | //(Tick list based on predefined acceptable ranges. Cross out if not applicable.)// | ||
| 48 | )))|(% style="width:1053px" %)((( | ||
| 49 | ☐ C3I operational, describe: ....................................................................................................................................................................................................................... | ||
| 50 | |||
| 51 | ☐ Health & location sensors connected, describe: ..................................................................................................................................................................................... | ||
| 52 | |||
| 53 | ☐ Acceptable/ baseline health ranges defined (HR, temperature, etc.), describe: ..................................................................................................................................... | ||
| 54 | |||
| 55 | ☐ OWL operational, describe: ..................................................................................................................................................................................................................... | ||
| 56 | |||
| 57 | ☐ ANYmal operational, describe: ................................................................................................................................................................................................................ | ||
| 58 | |||
| 59 | ☐ ANYmal with robot arm operational, describe: ........................................................................................................................................................................................ | ||
| 60 | |||
| 61 | ☐ ANYmal with SNAKE, describe: .............................................................................................................................................................................................................. | ||
| 62 | ))) | ||
| 63 | |(% colspan="2" %)**PRE-ENTRY PHASE (after briefing; before actual start of scenario)** | ||
| 64 | |(% style="width:296px" %)Hand out questionnaires|(% style="width:1053px" %)Each role their own question for this section | ||
| 65 | |(% style="width:296px" %)((( | ||
| 66 | Start time of scenario | ||
| 67 | )))|(% style="width:1053px" %)((( | ||
| 68 | **Timestamp of start of the scenario:.....................................................................................** | ||
| 69 | ))) | ||
| 70 | |(% style="width:296px" %)((( | ||
| 71 | Threshold breached, alert and intervention record | ||
| 72 | )))|(% style="width:1053px" %)((( | ||
| 73 | Health threshold breached (timestamp + action taken) | ||
| 74 | |||
| 75 | Alert detection by system (timestamp + action taken) | ||
| 76 | |||
| 77 | Intervention started (timestamp + action taken) | ||
| 78 | ))) | ||
| 79 | |(% colspan="2" style="width:296px" %)**DURING ENTRY PHASE** | ||
| 80 | |(% style="width:296px" %)((( | ||
| 81 | (if squad leader gets a questionnaire after DM, before human entry; pause timer or enter timestamps) | ||
| 82 | )))|(% style="width:1053px" %)((( | ||
| 83 | Timestamp of entering:..................................................................................... | Timestamp of exiting:........................................................................................................... | ||
| 84 | ))) | ||
| 85 | |(% style="width:296px" %)((( | ||
| 86 | Time within building/ hazard zone (humans) | ||
| 87 | )))|(% style="width:1053px" %)((( | ||
| 88 | Timestamp of entering:..................................................................................... | Timestamp of exiting:........................................................................................................... | ||
| 89 | |||
| 90 | Timestamp of entering:..................................................................................... | Timestamp of exiting:........................................................................................................... | ||
| 91 | ))) | ||
| 92 | |(% style="width:296px" %)((( | ||
| 93 | Time within building/ hazard zone (robots) | ||
| 94 | )))|(% style="width:1053px" %)((( | ||
| 95 | Timestamp of entering:..................................................................................... | Timestamp of exiting:........................................................................................................... | ||
| 96 | |||
| 97 | Timestamp of entering:..................................................................................... | Timestamp of exiting:........................................................................................................... | ||
| 98 | ))) | ||
| 99 | |(% style="width:296px" %)((( | ||
| 100 | Events with timestamp | ||
| 101 | )))|(% style="width:1053px" %)((( | ||
| 102 | Timestamp of victim detected:......................................................................... | Timestamp of victim detected:............................................................................................. | ||
| 103 | |||
| 104 | Timestamp of victim detected:......................................................................... | Timestamp of victim detected:............................................................................................. | ||
| 105 | |||
| 106 | |||
| 107 | Timestamp of manual robot control takeover:.................................................. | Timestamp of manual robot control takeover:.................................................................... | ||
| 108 | |||
| 109 | Timestamp of manual robot control takeover:.................................................. | Timestamp of manual robot control takeover:.................................................................... | ||
| 110 | |||
| 111 | |||
| 112 | Timestamp of health sensor notification:......................................................... | Timestamp of health sensor notification:............................................................................ | ||
| 113 | |||
| 114 | Timestamp of health sensor notification:......................................................... | Timestamp of health sensor notification:............................................................................ | ||
| 115 | ))) | ||
| 116 | |(% style="width:296px" %)End time of scenario|(% style="width:1053px" %)**Timestamp of finishing the scenario:........................................................................................................** | ||
| 117 | |(% colspan="2" style="width:296px" %)**POST EXIT PHASE (hot wash)** | ||
| 118 | |(% style="width:296px" %)Give every role their questionnaire|(% style="width:1053px" %) | ||
| 119 | |(% style="width:296px" %)Plenary questions: |(% style="width:1053px" %)((( | ||
| 120 | |(% colspan="2" style="width:587px" %)Open questions | ||
| 121 | |(% colspan="2" style="width:587px" %)((( | ||
| 122 | What was the plan/intent for the mission, and what did you personally aim to achieve?// (1–2 sentences)// | ||
| 123 | |||
| 124 | |||
| 125 | |||
| 126 | ))) | ||
| 127 | |(% colspan="2" style="width:587px" %)((( | ||
| 128 | What went well and should be sustained? Give one concrete example. //(example + why it worked)// | ||
| 129 | |||
| 130 | |||
| 131 | |||
| 132 | ))) | ||
| 133 | |(% colspan="2" style="width:587px" %)((( | ||
| 134 | What did not go as intended? Give one concrete example and its impact. //(example + consequence)// | ||
| 135 | |||
| 136 | |||
| 137 | |||
| 138 | ))) | ||
| 139 | |(% colspan="2" style="width:587px" %)((( | ||
| 140 | Why did it happen? What were the main contributing factors? //(e.g., information/SA, communication, coordination, timing, tools/technology, environment)// | ||
| 141 | |||
| 142 | |||
| 143 | |||
| 144 | ))) | ||
| 145 | |(% colspan="2" style="width:587px" %)((( | ||
| 146 | What are the top 2 actionable improvements for next time (specific, doable)? //(Action + owner/role + when)// | ||
| 147 | |||
| 148 | |||
| 149 | |||
| 150 | ))) | ||
| 151 | |||
| 152 | |||
| 153 | ))) | ||
| 154 | |||
| 155 | (% class="table-bordered" %) | ||
| 156 | |(% colspan="2" %)((( | ||
| 157 | == Template: Squad/ Team leader (Field) == | ||
| 158 | ))) | ||
| 159 | |(% colspan="2" %)((( | ||
| 160 | **General: ** | ||
| 161 | |||
| 162 | //Trial type/ Building/ scenario ID//: ................................. | //Date/ Time//: ....................................... | //Condition (encircle)//: Baseline / SYNERGISE-tech supported | ||
| 163 | ))) | ||
| 164 | |(% colspan="2" style="width:296px" %)**PRE-ENTRY PHASE (after briefing; before actual start of scenario)** | ||
| 165 | |(% style="width:296px" %)Baseline robot/ automation experience|(% style="width:1053px" %)((( | ||
| 166 | |Statement | ||
| 167 | |((( | ||
| 168 | I have experience with robot(s)/ automation in my daily work (encircle): Yes / No | ||
| 169 | |||
| 170 | Write down which (optional) | ||
| 171 | |||
| 172 | |||
| 173 | |||
| 174 | ))) | ||
| 175 | |((( | ||
| 176 | I have experience with robot(s)/ automation within the SYNERGISE project (encircle): Yes / No | ||
| 177 | |||
| 178 | Write down which (optional) | ||
| 179 | |||
| 180 | |||
| 181 | |||
| 182 | ))) | ||
| 183 | ))) | ||
| 184 | |(% style="width:296px" %)((( | ||
| 185 | (CL01) Safety Plan Confirmation | ||
| 186 | )))|(% style="width:1053px" %)((( | ||
| 187 | Mark chosen entry path on provided building map. //Sketch or describe entry path. (Sketch space)// | ||
| 188 | |||
| 189 | |||
| 190 | |||
| 191 | |||
| 192 | |||
| 193 | ))) | ||
| 194 | |(% style="width:296px" %)(CL02) Situation Awareness (SAGAT; Endsley 1988):|(% style="width:1053px" %)((( | ||
| 195 | List expected hazards and victims with details (e.g. location, gender, age, status). //(Bullets and/ or sketch)// | ||
| 196 | |||
| 197 | |||
| 198 | |||
| 199 | |||
| 200 | ))) | ||
| 201 | |(% style="width:296px" %)((( | ||
| 202 | (CL02) Situation Awareness interior (SAGAT; Endsley 1988): | ||
| 203 | |||
| 204 | |||
| 205 | )))|(% style="width:1053px" %)((( | ||
| 206 | Sketch/ Describe expected interior layout. //(Sketch space)// | ||
| 207 | |||
| 208 | |||
| 209 | |||
| 210 | |||
| 211 | |||
| 212 | ))) | ||
| 213 | |(% style="width:296px" %)(CL04) Trust in automation (Baseline (% style="background-color:transparent" %)Scale of Trust in Automated Systems; Jian et al. 2000). IF SYNERGISE TECH SCENARIO|(% style="width:1053px" %)((( | ||
| 214 | |Statement|1 (LOW)|2|3|4|5|6|7 (HIGH) | ||
| 215 | |I am confident in the robot/system’s abilities.|☐|☐|☐|☐|☐|☐|☐ | ||
| 216 | |The robot/system will act in our best interest.|☐|☐|☐|☐|☐|☐|☐ | ||
| 217 | |The robot/system is reliable.|☐|☐|☐|☐|☐|☐|☐ | ||
| 218 | |I trust the system’s information.|☐|☐|☐|☐|☐|☐|☐ | ||
| 219 | ))) | ||
| 220 | |(% colspan="2" style="width:296px" %)**HUMAN ENTRY PHASE** | ||
| 221 | |(% style="width:296px" %)(CL02 optional; if pause is feasable) Situation Awareness (mid-mission SAGAT probe; Endsley 1988)|(% style="width:1053px" %)((( | ||
| 222 | Where are you team members now and what hazards are near them? //(Sketch space)// | ||
| 223 | |||
| 224 | |||
| 225 | |||
| 226 | |||
| 227 | |||
| 228 | ))) | ||
| 229 | |(% colspan="2" style="width:296px" %)**POST EXIT PHASE (hot wash)** | ||
| 230 | |(% style="width:296px" %)((( | ||
| 231 | (CL01) Safety Decision Quality Rating (AAR-based; U.S. Army, 1993): | ||
| 232 | )))|(% style="width:1053px" %)((( | ||
| 233 | |(% style="width:587px" %)Statement|(% style="width:138px" %)1 (LOW)|(% style="width:60px" %)2|(% style="width:57px" %)3|(% style="width:59px" %)4|(% style="width:58px" %)5|(% style="width:60px" %)6|(% style="width:147px" %)7 (HIGH) | ||
| 234 | |(% style="width:587px" %)((( | ||
| 235 | I had a clear understanding of the mission objectives, my role, and the intended plan before execution. | ||
| 236 | )))|(% style="width:138px" %)☐|(% style="width:60px" %)☐|(% style="width:57px" %)☐|(% style="width:59px" %)☐|(% style="width:58px" %)☐|(% style="width:60px" %)☐|(% style="width:147px" %)☐ | ||
| 237 | |(% style="width:587px" %)((( | ||
| 238 | During the mission, I had sufficient situational awareness to understand what was happening and how the situation was evolving. | ||
| 239 | )))|(% style="width:138px" %)☐|(% style="width:60px" %)☐|(% style="width:57px" %)☐|(% style="width:59px" %)☐|(% style="width:58px" %)☐|(% style="width:60px" %)☐|(% style="width:147px" %)☐ | ||
| 240 | |(% style="width:587px" %)((( | ||
| 241 | The decisions made (by myself and/or the team) during the mission were timely and appropriate given the situation. | ||
| 242 | )))|(% style="width:138px" %)☐|(% style="width:60px" %)☐|(% style="width:57px" %)☐|(% style="width:59px" %)☐|(% style="width:58px" %)☐|(% style="width:60px" %)☐|(% style="width:147px" %)☐ | ||
| 243 | |(% style="width:587px" %)((( | ||
| 244 | Team coordination and communication were effective and supported successful task execution. | ||
| 245 | )))|(% style="width:138px" %)☐|(% style="width:60px" %)☐|(% style="width:57px" %)☐|(% style="width:59px" %)☐|(% style="width:58px" %)☐|(% style="width:60px" %)☐|(% style="width:147px" %)☐ | ||
| 246 | |(% style="width:587px" %)((( | ||
| 247 | The tools, systems, or support aids available during the mission effectively supported mission execution. | ||
| 248 | )))|(% style="width:138px" %)☐|(% style="width:60px" %)☐|(% style="width:57px" %)☐|(% style="width:59px" %)☐|(% style="width:58px" %)☐|(% style="width:60px" %)☐|(% style="width:147px" %)☐ | ||
| 249 | |(% style="width:587px" %)((( | ||
| 250 | Based on this exercise, the team is better prepared to perform similar missions in the future. | ||
| 251 | )))|(% style="width:138px" %)☐|(% style="width:60px" %)☐|(% style="width:57px" %)☐|(% style="width:59px" %)☐|(% style="width:58px" %)☐|(% style="width:60px" %)☐|(% style="width:147px" %)☐ | ||
| 252 | ))) | ||
| 253 | |(% style="width:296px" %)((( | ||
| 254 | (CL01) Safety Error Self-Assessment | ||
| 255 | )))|(% style="width:1053px" %)((( | ||
| 256 | List any unsafe or mistaken decisions you recognize from this mission (if any). //(Provide brief descriptions)// | ||
| 257 | |||
| 258 | |||
| 259 | |||
| 260 | ))) | ||
| 261 | |(% style="width:296px" %)((( | ||
| 262 | (CL02) Situation Awareness (Post-Scenario SAGAT) | ||
| 263 | )))|(% style="width:1053px" %)((( | ||
| 264 | From memory, sketch the interior layout and mark all hazards and victims encountered with list of details (e.g. location, gender, age, status). //(Sketch space and bullets)// | ||
| 265 | |||
| 266 | |||
| 267 | |||
| 268 | |||
| 269 | ))) | ||
| 270 | |(% style="width:296px" %)(CL03) Workload (NASA-TLX; Hart & Staveland 1988)|(% style="width:1053px" %)((( | ||
| 271 | |(% style="width:861px" %)Statement|(% style="width:305px" %)((( | ||
| 272 | Give a score between: | ||
| 273 | |||
| 274 | 0 (very low) - 100 (very high) | ||
| 275 | ))) | ||
| 276 | |(% style="width:861px" %)Mental demand - How much mental and perceptual activity was required - thinking, deciding, calculating, searching? Was the task easy or demanding, simple or complex, exacting or forgiving?|(% style="width:305px" %)[[image:1776690776402-253.png]] | ||
| 277 | |(% style="width:861px" %)Physical demand - How much physical activity was required - pushing, pulling, turning? Was the task easy or demanding, slow or brisk, restful or laborious?|(% style="width:305px" %)[[image:1776690777988-282.png]] | ||
| 278 | |(% style="width:861px" %)Temporal demand - How much time pressure did you feel? Was the pace slow and leisurely or rapid and frantic|(% style="width:305px" %)[[image:1776690778813-812.png]] | ||
| 279 | |(% style="width:861px" %)Effort - How hard did you work mentally and physically to accomplish your level of performance?|(% style="width:305px" %)[[image:1776690779609-960.png]] | ||
| 280 | |(% style="width:861px" %)Performance - How successful did you feel in accomplishing the goals of the task? How satisfied were you with your performance?|(% style="width:305px" %)[[image:1776690780664-359.png]] | ||
| 281 | |(% style="width:861px" %)Frustration level - How discouraged, stressed, irritated, and annoyed versus gratified, content, and relaxed did you feel during the task?|(% style="width:305px" %)[[image:1776690781527-833.png]] | ||
| 282 | ))) | ||
| 283 | |(% style="width:296px" %)((( | ||
| 284 | (CL04) Trust (Post-Use Trust in Automation scale; Jian et al. 2000) IF SYNERGISE TECH SCENARIO | ||
| 285 | )))|(% style="width:1053px" %)((( | ||
| 286 | |Statement|1 (LOW)|2|3|4|5|6|7 (HIGH) | ||
| 287 | |I would trust this robot system in future missions.|☐|☐|☐|☐|☐|☐|☐ | ||
| 288 | |I am confident in the robot/system’s abilities.|☐|☐|☐|☐|☐|☐|☐ | ||
| 289 | |The robot/system will act in our best interest.|☐|☐|☐|☐|☐|☐|☐ | ||
| 290 | |The robot/system is reliable.|☐|☐|☐|☐|☐|☐|☐ | ||
| 291 | |I trust the system’s information.|☐|☐|☐|☐|☐|☐|☐ | ||
| 292 | ))) | ||
| 293 | |||
| 294 | == == | ||
| 295 | |||
| 296 | (% class="table-bordered" %) | ||
| 297 | |(% colspan="2" %)((( | ||
| 298 | == Template: Robot operator == | ||
| 299 | ))) | ||
| 300 | |(% colspan="2" %)((( | ||
| 301 | **General: ** | ||
| 302 | |||
| 303 | //Trial type/ Building/ scenario ID//: ................................. | //Date/ Time//: ....................................... | //Condition (encircle)//: --Baseline --/ SYNERGISE-tech supported | ||
| 304 | ))) | ||
| 305 | |(% colspan="2" style="width:296px" %)**PRE-ENTRY PHASE (after briefing; before actual start of scenario)** | ||
| 306 | |(% style="width:296px" %)Baseline robot/ automation experience|(% style="width:1053px" %)((( | ||
| 307 | |Statement | ||
| 308 | |((( | ||
| 309 | I have experience with robot(s)/ automation in my daily work (encircle): Yes / No | ||
| 310 | |||
| 311 | Write down which (optional) | ||
| 312 | |||
| 313 | |||
| 314 | |||
| 315 | ))) | ||
| 316 | |((( | ||
| 317 | I have experience with robot(s)/ automation within the SYNERGISE project (encircle): Yes / No | ||
| 318 | |||
| 319 | Write down which (optional) | ||
| 320 | |||
| 321 | |||
| 322 | |||
| 323 | ))) | ||
| 324 | ))) | ||
| 325 | |(% style="width:296px" %)(CL04) Trust in automation (Baseline (% style="background-color:transparent" %)Scale of Trust in Automated Systems; Jian et al. 2000). |(% style="width:1053px" %)((( | ||
| 326 | |Statement|1 (LOW)|2|3|4|5|6|7 (HIGH) | ||
| 327 | |I am confident in the robot/system’s abilities.|☐|☐|☐|☐|☐|☐|☐ | ||
| 328 | |The robot/system will act in our best interest.|☐|☐|☐|☐|☐|☐|☐ | ||
| 329 | |The robot/system is reliable.|☐|☐|☐|☐|☐|☐|☐ | ||
| 330 | |I trust the system’s information.|☐|☐|☐|☐|☐|☐|☐ | ||
| 331 | ))) | ||
| 332 | |(% colspan="2" style="width:296px" %)**AFTER ROBOT entry** | ||
| 333 | |(% style="width:296px" %)((( | ||
| 334 | (CL02) Situation Awareness Robot Reconnaissance data understanding (OPTIONAL; is more important for the analyst) | ||
| 335 | )))|(% style="width:1053px" %)((( | ||
| 336 | After the robot’s initial run but before humans enter, list any //hazards, obstacles, or victims// that you (as operator) detected or suspect from the robot’s feed with list of details (e.g. location, gender, age, status). //(Open list)// | ||
| 337 | |||
| 338 | |||
| 339 | |||
| 340 | |||
| 341 | ))) | ||
| 342 | |(% colspan="2" style="width:296px" %)**POST EXIT PHASE (hot wash)** | ||
| 343 | |(% style="width:296px" %)((( | ||
| 344 | (CL01) SafetyTeam Decision Appraisal: | ||
| 345 | )))|(% style="width:1053px" %)((( | ||
| 346 | |Statement|1 (LOW)|2|3|4|5|6|7 (HIGH) | ||
| 347 | |How could you rate the team's decisions during the mission (including the robot) in terms of safety and effectiveness?|☐|☐|☐|☐|☐|☐|☐ | ||
| 348 | |Did your team correctly decide when to send the robot versus humans?|☐|☐|☐|☐|☐|☐|☐ | ||
| 349 | ))) | ||
| 350 | |(% style="width:296px" %)((( | ||
| 351 | (CL01) Safety Error Reflection | ||
| 352 | )))|(% style="width:1053px" %)((( | ||
| 353 | List any mistakes or unsafe decisions you think occurred regarding robot deployment. //(Open question)// | ||
| 354 | |||
| 355 | |||
| 356 | |||
| 357 | |||
| 358 | ))) | ||
| 359 | |(% style="width:296px" %)((( | ||
| 360 | (CL02) Situation Awareness (Post-Scenario SAGAT) | ||
| 361 | )))|(% style="width:1053px" %)((( | ||
| 362 | Describe what the robot observed/ found during the mission (e.g. hazards, obstacles) for navigation purposes. //(Sketch space and bullets)// | ||
| 363 | |||
| 364 | |||
| 365 | |||
| 366 | |||
| 367 | ))) | ||
| 368 | |(% style="width:296px" %)(CL03) Workload (NASA-TLX; Hart & Staveland 1988)|(% style="width:1053px" %)((( | ||
| 369 | |(% style="width:861px" %)Statement|(% style="width:305px" %)((( | ||
| 370 | Give a score between: | ||
| 371 | |||
| 372 | 0 (very low) - 100 (very high) | ||
| 373 | ))) | ||
| 374 | |(% style="width:861px" %)Mental demand - How much mental and perceptual activity was required - thinking, deciding, calculating, searching? Was the task easy or demanding, simple or complex, exacting or forgiving?|(% style="width:305px" %)[[image:1776690776402-253.png]] | ||
| 375 | |(% style="width:861px" %)Physical demand - How much physical activity was required - pushing, pulling, turning? Was the task easy or demanding, slow or brisk, restful or laborious?|(% style="width:305px" %)[[image:1776690777988-282.png]] | ||
| 376 | |(% style="width:861px" %)Temporal demand - How much time pressure did you feel? Was the pace slow and leisurely or rapid and frantic|(% style="width:305px" %)[[image:1776690778813-812.png]] | ||
| 377 | |(% style="width:861px" %)Effort - How hard did you work mentally and physically to accomplish your level of performance?|(% style="width:305px" %)[[image:1776690779609-960.png]] | ||
| 378 | |(% style="width:861px" %)Performance - How successful did you feel in accomplishing the goals of the task? How satisfied were you with your performance?|(% style="width:305px" %)[[image:1776690780664-359.png]] | ||
| 379 | |(% style="width:861px" %)Frustration level - How discouraged, stressed, irritated, and annoyed versus gratified, content, and relaxed did you feel during the task?|(% style="width:305px" %)[[image:1776690781527-833.png]] | ||
| 380 | ))) | ||
| 381 | |(% style="width:296px" %)((( | ||
| 382 | (CL04) Trust (Post-Use Trust in Automation scale; Jian et al. 2000) IF SYNERGISE TECH SCENARIO | ||
| 383 | )))|(% style="width:1053px" %)((( | ||
| 384 | |Statement|1 (LOW)|2|3|4|5|6|7 (HIGH) | ||
| 385 | |I would trust this robot system in future missions.|☐|☐|☐|☐|☐|☐|☐ | ||
| 386 | |I am confident in the robot/system’s abilities.|☐|☐|☐|☐|☐|☐|☐ | ||
| 387 | |The robot/system will act in our best interest.|☐|☐|☐|☐|☐|☐|☐ | ||
| 388 | |The robot/system is reliable.|☐|☐|☐|☐|☐|☐|☐ | ||
| 389 | |I trust the system’s information.|☐|☐|☐|☐|☐|☐|☐ | ||
| 390 | ))) | ||
| 391 | |||
| 392 | |||
| 393 | |||
| 394 | (% class="table-bordered" %) | ||
| 395 | |(% colspan="2" %)((( | ||
| 396 | == Template: Robot analyst == | ||
| 397 | ))) | ||
| 398 | |(% colspan="2" %)((( | ||
| 399 | **General: ** | ||
| 400 | |||
| 401 | //Trial type/ Building/ scenario ID//: ................................. | //Date/ Time//: ....................................... | //Condition (encircle)//: --Baseline --/ SYNERGISE-tech supported | ||
| 402 | ))) | ||
| 403 | |(% colspan="2" style="width:296px" %)**PRE-ENTRY PHASE (after briefing; before actual start of scenario)** | ||
| 404 | |(% style="width:296px" %)Baseline robot/ automation experience|(% style="width:1053px" %)((( | ||
| 405 | |Statement | ||
| 406 | |((( | ||
| 407 | I have experience with robot(s)/ automation in my daily work (encircle): Yes / No | ||
| 408 | |||
| 409 | Write down which (optional) | ||
| 410 | |||
| 411 | |||
| 412 | |||
| 413 | ))) | ||
| 414 | |((( | ||
| 415 | I have experience with robot(s)/ automation within the SYNERGISE project (encircle): Yes / No | ||
| 416 | |||
| 417 | Write down which (optional) | ||
| 418 | |||
| 419 | |||
| 420 | |||
| 421 | ))) | ||
| 422 | ))) | ||
| 423 | |(% style="width:445px" %)((( | ||
| 424 | (CL02) Situation Awareness Robot Reconnaissance Assessment (SAGAT expected situation report) | ||
| 425 | )))|(% style="width:1071px" %)((( | ||
| 426 | Based on the robot’s pre-entry scan, list the hazards, victims, and notable features you have identified or expect inside. //(Bullets and/or sketch)// | ||
| 427 | |||
| 428 | |||
| 429 | |||
| 430 | |||
| 431 | ))) | ||
| 432 | |(% style="width:296px" %)(CL04) Trust in automation (Baseline (% style="background-color:transparent" %)Scale of Trust in Automated Systems; Jian et al. 2000). |(% style="width:1053px" %)((( | ||
| 433 | |Statement|1 (LOW)|2|3|4|5|6|7 (HIGH) | ||
| 434 | |I am confident in the robot/system’s abilities.|☐|☐|☐|☐|☐|☐|☐ | ||
| 435 | |The robot/system will act in our best interest.|☐|☐|☐|☐|☐|☐|☐ | ||
| 436 | |The robot/system is reliable.|☐|☐|☐|☐|☐|☐|☐ | ||
| 437 | |I trust the system’s information.|☐|☐|☐|☐|☐|☐|☐ | ||
| 438 | ))) | ||
| 439 | |(% colspan="2" style="width:445px" %)**POST EXIT PHASE (hot wash)** | ||
| 440 | |(% style="width:296px" %)((( | ||
| 441 | (CL01) SafetyTeam Decision Appraisal: | ||
| 442 | )))|(% style="width:1053px" %)((( | ||
| 443 | |Statement|1 (LOW)|2|3|4|5|6|7 (HIGH) | ||
| 444 | |How could you rate the team's decisions during the mission (including the robot) in terms of safety and effectiveness?|☐|☐|☐|☐|☐|☐|☐ | ||
| 445 | |Did your team correctly decide when to send the robot versus humans?|☐|☐|☐|☐|☐|☐|☐ | ||
| 446 | ))) | ||
| 447 | |(% style="width:296px" %)((( | ||
| 448 | (CL01) Safety Error Reflection | ||
| 449 | )))|(% style="width:1053px" %)((( | ||
| 450 | List any mistakes or unsafe decisions you think occurred regarding robot deployment. //(Open question)// | ||
| 451 | |||
| 452 | |||
| 453 | |||
| 454 | |||
| 455 | ))) | ||
| 456 | |(% style="width:296px" %)((( | ||
| 457 | (CL02) Situation Awareness (Post-Scenario SAGAT) | ||
| 458 | )))|(% style="width:1053px" %)((( | ||
| 459 | Now that the mission is over, summarize the final floor layout, and list all hazards and victims found or confirmed. //(Sketch space and bullets)// | ||
| 460 | |||
| 461 | |||
| 462 | |||
| 463 | |||
| 464 | ))) | ||
| 465 | |(% style="width:296px" %)(CL03) Workload (NASA-TLX; Hart & Staveland 1988)|(% style="width:1053px" %)((( | ||
| 466 | |(% style="width:861px" %)Statement|(% style="width:305px" %)((( | ||
| 467 | Give a score between: | ||
| 468 | |||
| 469 | 0 (very low) - 100 (very high) | ||
| 470 | ))) | ||
| 471 | |(% style="width:861px" %)Mental demand - How much mental and perceptual activity was required - thinking, deciding, calculating, searching? Was the task easy or demanding, simple or complex, exacting or forgiving?|(% style="width:305px" %)[[image:1776690776402-253.png]] | ||
| 472 | |(% style="width:861px" %)Physical demand - How much physical activity was required - pushing, pulling, turning? Was the task easy or demanding, slow or brisk, restful or laborious?|(% style="width:305px" %)[[image:1776690777988-282.png]] | ||
| 473 | |(% style="width:861px" %)Temporal demand - How much time pressure did you feel? Was the pace slow and leisurely or rapid and frantic|(% style="width:305px" %)[[image:1776690778813-812.png]] | ||
| 474 | |(% style="width:861px" %)Effort - How hard did you work mentally and physically to accomplish your level of performance?|(% style="width:305px" %)[[image:1776690779609-960.png]] | ||
| 475 | |(% style="width:861px" %)Performance - How successful did you feel in accomplishing the goals of the task? How satisfied were you with your performance?|(% style="width:305px" %)[[image:1776690780664-359.png]] | ||
| 476 | |(% style="width:861px" %)Frustration level - How discouraged, stressed, irritated, and annoyed versus gratified, content, and relaxed did you feel during the task?|(% style="width:305px" %)[[image:1776690781527-833.png]] | ||
| 477 | ))) | ||
| 478 | |(% style="width:296px" %)((( | ||
| 479 | (CL04) Trust (Post-Use Trust in Automation scale; Jian et al. 2000) | ||
| 480 | )))|(% style="width:1053px" %)((( | ||
| 481 | |Statement|1 (LOW)|2|3|4|5|6|7 (HIGH) | ||
| 482 | |I would trust this robot system in future missions.|☐|☐|☐|☐|☐|☐|☐ | ||
| 483 | |I am confident in the robot/system’s abilities.|☐|☐|☐|☐|☐|☐|☐ | ||
| 484 | |The robot/system will act in our best interest.|☐|☐|☐|☐|☐|☐|☐ | ||
| 485 | |The robot/system is reliable.|☐|☐|☐|☐|☐|☐|☐ | ||
| 486 | |I trust the system’s information.|☐|☐|☐|☐|☐|☐|☐ | ||
| 487 | ))) | ||
| 488 | |||
| 489 | |||
| 490 | |||
| 491 | (% class="table-bordered" %) | ||
| 492 | |(% colspan="2" %)((( | ||
| 493 | == Template: Entry team member == | ||
| 494 | ))) | ||
| 495 | |(% colspan="2" %)((( | ||
| 496 | **General: ** | ||
| 497 | |||
| 498 | //Trial type/ Building/ scenario ID//: ................................. | //Date/ Time//: ....................................... | //Condition (encircle)//: Baseline / SYNERGISE-tech supported | ||
| 499 | ))) | ||
| 500 | |(% colspan="2" style="width:296px" %)**PRE-ENTRY PHASE (after briefing; before actual start of scenario)** | ||
| 501 | |(% style="width:296px" %)Baseline robot/ automation experience|(% style="width:1053px" %)((( | ||
| 502 | |Statement | ||
| 503 | |((( | ||
| 504 | I have experience with robot(s)/ automation in my daily work (encircle): Yes / No | ||
| 505 | |||
| 506 | Write down which (optional) | ||
| 507 | |||
| 508 | |||
| 509 | |||
| 510 | ))) | ||
| 511 | |((( | ||
| 512 | I have experience with robot(s)/ automation within the SYNERGISE project (encircle): Yes / No | ||
| 513 | |||
| 514 | Write down which (optional) | ||
| 515 | |||
| 516 | |||
| 517 | |||
| 518 | ))) | ||
| 519 | ))) | ||
| 520 | |(% style="width:296px" %)((( | ||
| 521 | (CL02) Situation Awareness (Pre-mission knowledge check): | ||
| 522 | )))|(% style="width:1053px" %)((( | ||
| 523 | Before entering, list the hazards or victims you expect to encounter, based on briefing info. //(Bullets)// | ||
| 524 | |||
| 525 | |||
| 526 | |||
| 527 | ))) | ||
| 528 | |(% style="width:296px" %)(CL04) Trust in automation (Baseline (% style="background-color:transparent" %)Scale of Trust in Automated Systems; Jian et al. 2000). IF SYNERGISE TECH SCENARIO|(% style="width:1053px" %)((( | ||
| 529 | |Statement|1 (LOW)|2|3|4|5|6|7 (HIGH) | ||
| 530 | |I am confident in the robot analyst's capabilities.|☐|☐|☐|☐|☐|☐|☐ | ||
| 531 | |The robot/system will act in our best interest.|☐|☐|☐|☐|☐|☐|☐ | ||
| 532 | |The robot/system is reliable.|☐|☐|☐|☐|☐|☐|☐ | ||
| 533 | |I trust the system’s information.|☐|☐|☐|☐|☐|☐|☐ | ||
| 534 | ))) | ||
| 535 | |(% colspan="2" style="width:296px" %)**POST EXIT PHASE (hot wash)** | ||
| 536 | |(% style="width:296px" %)((( | ||
| 537 | (CL01) Safety - Team Decision Appraisal: | ||
| 538 | )))|(% style="width:1053px" %)((( | ||
| 539 | |Statement|1 (LOW)|2|3|4|5|6|7 (HIGH) | ||
| 540 | |How would you rate the team's decisions during the mission (including the robot) in terms of safety and effectiveness?|☐|☐|☐|☐|☐|☐|☐ | ||
| 541 | |Did your team correctly decide when to send the robot versus humans?|☐|☐|☐|☐|☐|☐|☐ | ||
| 542 | ))) | ||
| 543 | |(% style="width:296px" %)((( | ||
| 544 | (CL02) Situation Awareness Post Operation Recall | ||
| 545 | )))|(% style="width:1053px" %)((( | ||
| 546 | Describe the layout and list all hazards/victims that were found. //(Sketch and describe)// | ||
| 547 | |||
| 548 | |||
| 549 | |||
| 550 | |||
| 551 | |||
| 552 | ))) | ||
| 553 | |(% style="width:296px" %)(CL03) Workload (NASA-TLX; Hart & Staveland 1988)|(% style="width:1053px" %)((( | ||
| 554 | |(% style="width:861px" %)Statement|(% style="width:305px" %)((( | ||
| 555 | Give a score between: | ||
| 556 | |||
| 557 | 0 (very low) - 100 (very high) | ||
| 558 | ))) | ||
| 559 | |(% style="width:861px" %)Mental demand - How much mental and perceptual activity was required - thinking, deciding, calculating, searching? Was the task easy or demanding, simple or complex, exacting or forgiving?|(% style="width:305px" %)[[image:1776690776402-253.png]] | ||
| 560 | |(% style="width:861px" %)Physical demand - How much physical activity was required - pushing, pulling, turning? Was the task easy or demanding, slow or brisk, restful or laborious?|(% style="width:305px" %)[[image:1776690777988-282.png]] | ||
| 561 | |(% style="width:861px" %)Temporal demand - How much time pressure did you feel? Was the pace slow and leisurely or rapid and frantic|(% style="width:305px" %)[[image:1776690778813-812.png]] | ||
| 562 | |(% style="width:861px" %)Effort - How hard did you work mentally and physically to accomplish your level of performance?|(% style="width:305px" %)[[image:1776690779609-960.png]] | ||
| 563 | |(% style="width:861px" %)Performance - How successful did you feel in accomplishing the goals of the task? How satisfied were you with your performance?|(% style="width:305px" %)[[image:1776690780664-359.png]] | ||
| 564 | |(% style="width:861px" %)Frustration level - How discouraged, stressed, irritated, and annoyed versus gratified, content, and relaxed did you feel during the task?|(% style="width:305px" %)[[image:1776690781527-833.png]] | ||
| 565 | ))) | ||
| 566 | |(% style="width:296px" %)((( | ||
| 567 | (CL04) Trust (Post-Use Trust in Automation scale; Jian et al. 2000) IF SYNERGISE TECH SCENARIO | ||
| 568 | )))|(% style="width:1053px" %)((( | ||
| 569 | |Statement|1 (LOW)|2|3|4|5|6|7 (HIGH) | ||
| 570 | |I would trust this robot system in future missions.|☐|☐|☐|☐|☐|☐|☐ | ||
| 571 | |I am confident in the robot/system’s abilities.|☐|☐|☐|☐|☐|☐|☐ | ||
| 572 | |The robot/system will act in our best interest.|☐|☐|☐|☐|☐|☐|☐ | ||
| 573 | |The robot/system is reliable.|☐|☐|☐|☐|☐|☐|☐ | ||
| 574 | |I trust the system’s information.|☐|☐|☐|☐|☐|☐|☐ | ||
| 575 | ))) | ||
| 576 | |||
| 577 | |||
| 578 | |||
| 579 | (% class="table-bordered" %) | ||
| 580 | |(% colspan="2" %)((( | ||
| 581 | == Template: Base of Operations (LEMA) == | ||
| 582 | ))) | ||
| 583 | |(% colspan="2" %)((( | ||
| 584 | **General: ** | ||
| 585 | |||
| 586 | //Trial type/ Building/ scenario ID//: ................................. | //Date/ Time//: ....................................... | //Condition (encircle)//: Baseline / SYNERGISE-tech supported | ||
| 587 | ))) | ||
| 588 | |(% colspan="2" style="width:296px" %)**PRE-ENTRY PHASE (after briefing; before actual start of scenario)** | ||
| 589 | |(% style="width:296px" %)Baseline robot/ automation experience|(% style="width:1053px" %)((( | ||
| 590 | |Statement | ||
| 591 | |((( | ||
| 592 | I have experience with robot(s)/ automation in my daily work (encircle): Yes / No | ||
| 593 | |||
| 594 | Write down which (optional) | ||
| 595 | |||
| 596 | |||
| 597 | |||
| 598 | ))) | ||
| 599 | |((( | ||
| 600 | I have experience with robot(s)/ automation within the SYNERGISE project (encircle): Yes / No | ||
| 601 | |||
| 602 | Write down which (optional) | ||
| 603 | |||
| 604 | |||
| 605 | |||
| 606 | ))) | ||
| 607 | ))) | ||
| 608 | |(% style="width:296px" %)(CL04) Trust in automation (Baseline (% style="background-color:transparent" %)Scale of Trust in Automated Systems; Jian et al. 2000). IF SYNERGISE TECH SCENARIO|(% style="width:1053px" %)((( | ||
| 609 | |Statement|1 (LOW)|2|3|4|5|6|7 (HIGH) | ||
| 610 | |I am confident in the robot analyst's capabilities.|☐|☐|☐|☐|☐|☐|☐ | ||
| 611 | |The robot/system will act in our best interest.|☐|☐|☐|☐|☐|☐|☐ | ||
| 612 | |The robot/system is reliable.|☐|☐|☐|☐|☐|☐|☐ | ||
| 613 | |I trust the system’s information.|☐|☐|☐|☐|☐|☐|☐ | ||
| 614 | ))) | ||
| 615 | |(% colspan="2" style="width:296px" %)**POST EXIT PHASE (hot wash)** | ||
| 616 | |(% style="width:296px" %)(CL01, CL03-05) Safety and Effectiveness|(% style="width:1053px" %)((( | ||
| 617 | |Statement|1 (LOW)|2|3|4|5|6|7 (HIGH) | ||
| 618 | |From the command (strategic) perspective, rate the overall mission outcome on safety and effectiveness.|☐|☐|☐|☐|☐|☐|☐ | ||
| 619 | ))) | ||
| 620 | |(% style="width:296px" %)((( | ||
| 621 | (CL04) Trust (Post-Use Trust in Automation scale; Jian et al. 2000) IF SYNERGISE TECH SCENARIO | ||
| 622 | )))|(% style="width:1053px" %)((( | ||
| 623 | |Statement|1 (LOW)|2|3|4|5|6|7 (HIGH) | ||
| 624 | |I would trust this robot system in future missions.|☐|☐|☐|☐|☐|☐|☐ | ||
| 625 | |I am confident in the robot/system’s abilities.|☐|☐|☐|☐|☐|☐|☐ | ||
| 626 | |The robot/system will act in our best interest.|☐|☐|☐|☐|☐|☐|☐ | ||
| 627 | |The robot/system is reliable.|☐|☐|☐|☐|☐|☐|☐ | ||
| 628 | |I trust the system’s information.|☐|☐|☐|☐|☐|☐|☐ | ||
| 629 | ))) | ||
| 630 | |(% style="width:296px" %)(CL03) Workload (NASA-TLX; Hart & Staveland 1988)|(% style="width:1053px" %)((( | ||
| 631 | |(% style="width:861px" %)Statement|(% style="width:305px" %)((( | ||
| 632 | Give a score between: | ||
| 633 | |||
| 634 | 0 (very low) - 100 (very high) | ||
| 635 | ))) | ||
| 636 | |(% style="width:861px" %)Mental demand - How much mental and perceptual activity was required - thinking, deciding, calculating, searching? Was the task easy or demanding, simple or complex, exacting or forgiving?|(% style="width:305px" %)[[image:1776690776402-253.png]] | ||
| 637 | |(% style="width:861px" %)Physical demand - How much physical activity was required - pushing, pulling, turning? Was the task easy or demanding, slow or brisk, restful or laborious?|(% style="width:305px" %)[[image:1776690777988-282.png]] | ||
| 638 | |(% style="width:861px" %)Temporal demand - How much time pressure did you feel? Was the pace slow and leisurely or rapid and frantic|(% style="width:305px" %)[[image:1776690778813-812.png]] | ||
| 639 | |(% style="width:861px" %)Effort - How hard did you work mentally and physically to accomplish your level of performance?|(% style="width:305px" %)[[image:1776690779609-960.png]] | ||
| 640 | |(% style="width:861px" %)Performance - How successful did you feel in accomplishing the goals of the task? How satisfied were you with your performance?|(% style="width:305px" %)[[image:1776690780664-359.png]] | ||
| 641 | |(% style="width:861px" %)Frustration level - How discouraged, stressed, irritated, and annoyed versus gratified, content, and relaxed did you feel during the task?|(% style="width:305px" %)[[image:1776690781527-833.png]] | ||
| 642 | ))) | ||
| 643 | |||
| 644 | === Before humans enter a building/ hazard === | ||
| 645 | |||
| 646 | |**Claim**|**Measurement**|**What is recorded**|**Who records** | ||
| 647 | |CL01 Safety|Ground-truth safe paths|predefined safe/ suboptimal / unsafe paths|//Evaluation leader// | ||
| 648 | |((( | ||
| 649 | CL01 Safety | ||
| 650 | )))|((( | ||
| 651 | Chosen entry path | ||
| 652 | )))|((( | ||
| 653 | Intended entry path selection | ||
| 654 | )))|//Squad/ Team leader (informed by Robot analyst)// | ||
| 655 | |((( | ||
| 656 | CL02 Situation Awareness | ||
| 657 | )))|((( | ||
| 658 | Reported hazards, victims, layout (pre) | ||
| 659 | )))|((( | ||
| 660 | Sketch / map / description of interior | ||
| 661 | )))|//Squad/ Team leader, Entry team, Robot analyst (separately)// | ||
| 662 | |((( | ||
| 663 | CL02 SA | ||
| 664 | )))|((( | ||
| 665 | Robot‑based expectations | ||
| 666 | )))|((( | ||
| 667 | Expected hazards / victims from robot data | ||
| 668 | )))|//Robot analyst// | ||
| 669 | |CL05 Trustworthy SA|Baseline trust (optional)|Initial trust in system/ robot|//Squad/ Team leader, Robot operator, Robot analyst// | ||
| 670 | |((( | ||
| 671 | CL06 Health | ||
| 672 | )))|((( | ||
| 673 | Acceptable health ranges | ||
| 674 | )))|((( | ||
| 675 | HR, temp thresholds | ||
| 676 | )))|//Evaluation leader// | ||
| 677 | |((( | ||
| 678 | CL07 Efficiency | ||
| 679 | )))|((( | ||
| 680 | Mission start timestamp | ||
| 681 | )))|((( | ||
| 682 | End briefing → start | ||
| 683 | )))|//Evaluation leader// | ||
| 684 | |||
| 685 | During humans/robots are inside building/ hazard | ||
| 686 | |||
| 687 | |**Claim**|**Measurement**|**What is recorded**|**Who records** | ||
| 688 | |((( | ||
| 689 | CL01 Safety | ||
| 690 | )))|((( | ||
| 691 | Near‑incidents | ||
| 692 | )))|((( | ||
| 693 | Unsafe situations / close calls | ||
| 694 | )))|//Evaluation leader// | ||
| 695 | |((( | ||
| 696 | CL01 Safety | ||
| 697 | )))|((( | ||
| 698 | Hazard avoidance | ||
| 699 | )))|((( | ||
| 700 | Avoided predefined hazards | ||
| 701 | )))|//Evaluation leader// | ||
| 702 | |((( | ||
| 703 | CL02 SA | ||
| 704 | )))|((( | ||
| 705 | Executed path | ||
| 706 | )))|((( | ||
| 707 | Actual path taken | ||
| 708 | )))|//Evaluation leader, Engage system/ location sensors (?)// | ||
| 709 | |((( | ||
| 710 | CL02 SA | ||
| 711 | )))|((( | ||
| 712 | Path deviations | ||
| 713 | )))|((( | ||
| 714 | Deviations + cause (something for after instead?) | ||
| 715 | )))|//Squad/ Team leader, Entry team// | ||
| 716 | |((( | ||
| 717 | CL02 SA | ||
| 718 | )))|((( | ||
| 719 | Interpretations of robot images | ||
| 720 | )))|((( | ||
| 721 | ...? | ||
| 722 | )))|//Robot analyst (and Base or Operations/ LEMA ?)// | ||
| 723 | |((( | ||
| 724 | CL02 SA (Optional) | ||
| 725 | )))|((( | ||
| 726 | Robot control actions | ||
| 727 | )))|((( | ||
| 728 | Overrides, manual interventions | ||
| 729 | )))|//Robot operator// | ||
| 730 | |((( | ||
| 731 | CL06 Health | ||
| 732 | )))|((( | ||
| 733 | Threshold breach | ||
| 734 | )))|((( | ||
| 735 | Breach moment (HR, temp, stress (?)) | ||
| 736 | )))|//Evaluation leader// | ||
| 737 | |((( | ||
| 738 | CL06 Health | ||
| 739 | )))|((( | ||
| 740 | Detection & intervention | ||
| 741 | )))|((( | ||
| 742 | Detection with corresponding action | ||
| 743 | )))|//Evaluation leader// | ||
| 744 | |((( | ||
| 745 | CL07 Efficiency | ||
| 746 | )))|((( | ||
| 747 | Entry timestamp | ||
| 748 | )))|((( | ||
| 749 | Entry team enters OR robot enters | ||
| 750 | )))|//Evaluation leader// | ||
| 751 | |((( | ||
| 752 | CL07 Efficiency | ||
| 753 | )))|((( | ||
| 754 | Exit timestamp | ||
| 755 | )))|Entry team exits OR robot exits |//Evaluation leader// | ||
| 756 | |||
| 757 | === After humans/robots exited building/hazard === | ||
| 758 | |||
| 759 | |**Claim**|**Measurement**|**What is recorded**|**Who records** | ||
| 760 | |((( | ||
| 761 | |||
| 762 | |||
| 763 | CL01 Safety | ||
| 764 | )))|((( | ||
| 765 | |||
| 766 | |||
| 767 | Decision quality (retro) | ||
| 768 | )))|((( | ||
| 769 | |||
| 770 | |||
| 771 | “How good were decisions?” | ||
| 772 | )))|//Squad/ Team leader, Entry team, Robot operator, Robot analyst, Baes of Operations/ LEMA// | ||
| 773 | |((( | ||
| 774 | CL01 Safety | ||
| 775 | )))|((( | ||
| 776 | Error reflection | ||
| 777 | )))|((( | ||
| 778 | Reflected unsafe choices | ||
| 779 | )))|//Squad/ Team leader, Robot operator, Robot analyst// | ||
| 780 | |((( | ||
| 781 | CL02 SA | ||
| 782 | )))|((( | ||
| 783 | Reported hazards, victims, layout (post) | ||
| 784 | )))|((( | ||
| 785 | Sketch / map / description | ||
| 786 | )))|//Squad/ Team leader, Entry team, Robot analyst// | ||
| 787 | |CL03 mission effectiveness|Decision and path quality|Route of FRs, decisions made.|//Squad/ Team leader, Entry team, Robot analyst// | ||
| 788 | |((( | ||
| 789 | CL04 acceptable workload | ||
| 790 | )))|((( | ||
| 791 | Workload | ||
| 792 | )))|((( | ||
| 793 | Short NASA‑TLX | ||
| 794 | )))|//Squad/ Team leader, Entry team, Robot operator, Robot analyst, Baes of Operations/ LEMA// | ||
| 795 | |((( | ||
| 796 | CL05 Trustworthy SA | ||
| 797 | )))|((( | ||
| 798 | Trust | ||
| 799 | )))|((( | ||
| 800 | Trust survey / interview | ||
| 801 | )))|//Squad/ Team leader, Entry team (?), Robot operator, Robot analyst, Baes of Operations/ LEMA// | ||
| 802 | |((( | ||
| 803 | CL06 Health | ||
| 804 | )))|((( | ||
| 805 | Health issue handling | ||
| 806 | )))|((( | ||
| 807 | Adequacy & timeliness | ||
| 808 | )))|//Evaluation leader (+ expert)// | ||
| 809 | |((( | ||
| 810 | CL07 Efficiency | ||
| 811 | )))|((( | ||
| 812 | Total mission duration | ||
| 813 | )))|((( | ||
| 814 | Start (end briefing) till end (?) | ||
| 815 | )))|//Evaluation leader// | ||
| 816 | |||
| 817 |