@conference {cTarresa, title = {Sign Language Translation from Instructional Videos}, booktitle = {CVPR 2023 Women in Computer Vision Workshop}, year = {2023}, month = {04/2023}, publisher = {Computer Vision Foundation / IEEE}, organization = {Computer Vision Foundation / IEEE}, address = {Vancouver, Canada}, abstract = {

The advances in automatic sign language translation (SLT) to spoken languages have been mostly benchmarked with datasets of limited size and restricted domains. Our work advances the state of the art by providing the first baseline results on How2Sign, a large and broad dataset. We train a Transformer over I3D video features, using the reduced BLEU as a reference metric for validation, instead of the widely used BLEU score. We report a result of 8.03 on the BLEU score, and publish the first open-source implementation of its kind to promote further advances.

}, author = {Laia Tarr{\'e}s and Gerard I. G{\'a}llego and Amanda Duarte and Jordi Torres and Xavier Gir{\'o}-i-Nieto} } @article {xCabot22, title = {Sign Language Translation based on Transformers for the How2Sign Dataset}, year = {2022}, abstract = {

Introduction to Research, BSc Data Science and Engineering, Autumn 2021:

The end goal of Sign Language Translation is to either produce spoken sentences from sign videos or generate sign videos from their corresponding written transcriptions. In this situation, this task has been address in multiple approaches in recent years. Moreover, it has been proved that taking advantage of the sign gloss representations improves substantially the model{\textquoteright}s performance in this task. Therefore, in this work we replicate the state-of-the-art Transformer-based approach on the task and evaluate it on the multimodal American Sign Language How2Sign dataset. Furthermore, we provide baseline recognition and translation results that represent an starting point to further research on the topic. In addition, we provide a new sentence-based alignment for the How2Sign videos, as their current alignment was with speech, which we have used to tackle the Sign Language Translation task properly.\ 

}, author = {Patricia Cabot}, editor = {Laia Tarr{\'e}s and Xavier Gir{\'o}-i-Nieto} } @mastersthesis {xCabot22a, title = {Sign-Language Translation with Pseudo-Glosses}, year = {2022}, abstract = {

Sign Language Translation is an open problem whose goal is to generate written sentences from sign videos. In recent years, many research works that have been developed in this field mainly addressed the Sign Language Recognition task, which consists in understanding the input signs and transcribing them into sequences of annotations. Moreover, current studies show that taking advantage of the latter task helps to learn meaningful representations and can be seen as an intermediate step towards the end goal of translation.


In this work, we present a method to generate automatic pseudo-glosses from written sentences, which can work as a replacement for real glosses. This addresses the issue of their collection, as they need to be manually annotated and it is extremely costly.

Furthermore, we introduce a new implementation built on Fairseq of the Transformer-model approach introduced by Camgoz et al., which is jointly trained to solve the recognition and translation tasks. Besides, we provide new baseline results on both implementations: first, on the Phoenix dataset, we present results that outperform the ones provided by Camgoz et al. in their work, and, second, on the How2Sign dataset, we present the first results on the translation task. These results can work as a baseline for future research in the field.

Patricia Cabot with her advisors Laia Tarr{\'e}s, Gerard I. G{\'a}llego and Xavier Gir{\'o}-i-Nieto.
}, author = {Patricia Cabot and Laia Tarr{\'e}s and Xavier Gir{\'o}-i-Nieto} }