@article{Liu2024, 
author = {Tianyu Liu and Fuchun Sun},
title = {Self-Aligning Multi-Modal Transformer for Oropharyngeal Swab Point Localization},
year = {2024},
journal = {Tsinghua Science and Technology},
volume = {29},
number = {4},
pages = {1082-1091},
keywords = {segmentation, localization, transformer, multi-modal perception, robotic perception},
url = {https://www.sciopen.com/article/10.26599/TST.2023.9010070},
doi = {10.26599/TST.2023.9010070},
abstract = {The oropharyngeal swabbing is a pre-diagnostic procedure used to test various respiratory diseases, including COVID and Influenza A (H1N1). To improve the testing efficiency of testing, a real-time, accurate, and robust sampling point localization algorithm is needed for robots. However, current solutions rely heavily on visual input, which is not reliable enough for large-scale deployment. The transformer has significantly improved the performance of image-related tasks and challenged the dominance of traditional convolutional neural networks (CNNs) in the image field. Inspired by its success, we propose a novel self-aligning multi-modal transformer (SAMMT) to dynamically attend to different parts of unaligned feature maps, preventing information loss caused by perspective disparity and simplifying overall implementation. Unlike preexisting multi-modal transformers, our attention mechanism works in image space instead of embedding space, rendering the need for the sensor registration process obsolete. To facilitate the multi-modal task, we collected and annotate an oropharynx localization/segmentation dataset by trained medical personnel. This dataset is open-sourced and can be used for future multi-modal research. Our experiments show that our model improves the performance of the localization task by 4.2% compared to the pure visual model, and reduces the pixel-wise error rate of the segmentation task by 16.7% compared to the CNN baseline.}
}