@phdthesis {dDuarte, title = {Data and methods for a visual understanding of sign languages}, volume = {PhD}, year = {2022}, type = {International Mention}, abstract = {

Signed languages are complete and natural languages used as the first or preferred mode of communication by millions of people worldwide. However, they, unfortunately, continue to be marginalized languages. Designing, building, and evaluating models that work on sign languages presents compelling research challenges and requires interdisciplinary and collaborative efforts. The recent advances in Machine Learning (ML) and Artificial Intelligence (AI) has the power to enable better accessibility to sign language users and narrow down the existing communication barrier between the Deaf community and non-sign language users. However, recent AI-powered technologies still do not account for sign language in their pipelines. This is mainly because sign languages are visual languages, that use manual and non-manual features to convey information, and do not have a standard written form. Thus, the goal of this thesis is to contribute to the development of new technologies that account for sign language by creating large-scale multimodal resources suitable for training modern data-hungry machine learning models and developing automatic systems that focus on computer vision tasks related to sign language that aims at learning better visual understanding of sign languages.Thus, in Part I, we introduce the How2Sign dataset, which is a large-scale collection of multimodal and multiview sign language videos in American Sign Language. In Part II, we contribute to the development of technologies that account for sign languages by presenting in Chapter 4 a framework called Spot-Align, based on sign spotting methods, to automatically annotate sign instances in continuous sign language. We further present the benefits of this framework and establish a baseline for the sign language recognition task on the How2Sign dataset. In addition to that, in Chapter 5 we benefit from the different annotations and modalities of the How2Sign to explore sign language video retrieval by learning cross-modal embeddings. Later in Chapter 6, we explore sign language video generation by applying Generative Adversarial Networks to the sign language domain and assess if and how well sign language users can understand automatically generated sign language videos by proposing an evaluation protocol based on How2Sign topics and English translation.

}, author = {Amanda Duarte}, editor = {Jordi Torres and Xavier Gir{\'o}-i-Nieto} } @phdthesis {dCampos20, title = {Deep Learning that Scales: Leveraging Compute and Data}, year = {2020}, month = {12/2020}, school = {Universitat Polit{\`e}cnica de Catalunya}, address = {Barcelona, Catalonia}, abstract = {

Deep learning has revolutionized the field of artificial intelligence in the past decade. Although the development of these techniques spans over several years, the recent advent of deep learning is explained by an increased availability of data and compute that have unlocked the potential of deep neural networks. They have become ubiquitous in domains such as natural language processing, computer vision, speech processing, and control, where enough training data is available. Recent years have seen continuous progress driven by ever-growing neural networks that benefited from large amounts of data and computing power. This thesis is motivated by the observation that scale is one of the key factors driving progress in deep learning research, and aims at devising deep learning methods that scale gracefully with the available data and compute. We narrow down this scope into two main research directions. The first of them is concerned with designing hardware-aware methods which can make the most of the computing resources in current high performance computing facilities. We then study bottlenecks preventing existing methods from scaling up as more data becomes available, providing solutions that contribute towards enabling training of more complex models. This dissertation studies the aforementioned research questions for two different learning paradigms, each with its own algorithmic and computational characteristics. The first part of this thesis studies the paradigm where the model needs to learn from a collection of examples, extracting as much information as possible from the given data. The second part is concerned with training agents that learn by interacting with a simulated environment, which introduces unique challenges such as efficient exploration and simulation.

}, url = { http://hdl.handle.net/10803/670372}, author = {V{\'\i}ctor Campos}, editor = {Jordi Torres and Xavier Gir{\'o}-i-Nieto} } @conference {cBellvera, title = {Detection-aided liver lesion segmentation using deep learning}, booktitle = {ML4H: Machine Learning for Health Workshop at NIPS 2017}, year = {2017}, month = {11/2017}, abstract = {

A fully automatic technique for segmenting the liver and localizing its unhealthy tissues is a convenient tool in order to diagnose hepatic diseases and assess the response to the according treatments. In this work we propose a method to segment the liver and its lesions from Computed Tomography (CT) scans using Convolutional Neural Networks (CNNs), that have proven good results in a variety of computer vision tasks, including medical imaging. The network that segments the lesions consists of a cascaded architecture, which first focuses on the region of the liver in order to segment the lesions on it. Moreover, we train a detector to localize the lesions, and mask the results of the segmentation network with the positive detections. The segmentation architecture is based on DRIU, a Fully Convolutional Network (FCN) with side outputs that work on feature maps of different resolutions, to finally\  benefit from the multi-scale information learned by different stages of the network. The main contribution of this work is the use of a detector to localize the lesions, which we show to be beneficial to remove false positives triggered by the segmentation network.

Detection-aided liver lesion segmentation using deep learning from Xavier Giro-i-Nieto
}, author = {M{\'\i}riam Bellver and Kevis-Kokitsi Maninis and Jordi Pont-Tuset and Jordi Torres and Xavier Gir{\'o}-i-Nieto and Luc van Gool} } @mastersthesis {xBellvera, title = {Detection-aided medical image segmentation using deep learning}, year = {2017}, abstract = {

Program:\ Master{\textquoteright}s Degree in Telecommunications Engineering

Grade: A with honours (10.0/10.0)

A fully automatic technique for segmenting the liver and localizing its unhealthy tissues is a convenient tool in order to diagnose hepatic diseases and also to assess the response to the according treatments. In this thesis we propose a method to segment the liver and its lesions from Computed Tomography (CT) scans, as well as other anatomical structures and organs of the human body. We have used Convolutional Neural Networks (CNNs), that have proven good results in a variety of tasks, including medical imaging. The network to segment the lesions consists of a cascaded architecture, which first focuses on the liver region in order to segment the lesion. Moreover, we train a detector to localize the lesions and just keep those pixels from the output of the segmentation network where a lesion is detected. The segmentation architecture is based on DRIU [24], a Fully Convolutional Network (FCN) with side outputs that work at feature maps of different resolutions, to finally benefit from the multi-scale information learned by different stages of the network. Our pipeline is 2.5D, as the input of the network is a stack of consecutive slices of the CT scans. We also study different methods to benefit from the liver segmentation in order to delineate the lesion. The main focus of this work is to use the detector to localize the lesions, as we demonstrate that it helps to remove false positives triggered by the segmentation network. The benefits of using a detector on top of the segmentation is that the detector acquires a more global insight of the healthiness of a liver tissue compared to the segmentation network, whose final output is pixel-wise and is not forced to take a global decision over a whole liver patch. We show experiments with the LiTS dataset for the lesion and liver segmentation. In order to prove the generality of the segmentation network, we also segment several anatomical structures from the Visceral dataset.

Detection-aided liver lesion segmentation using deep learning from Xavier Giro-i-Nieto
}, author = {M{\'\i}riam Bellver}, editor = {Kevis-Kokitsi Maninis and Jordi Pont-Tuset and Luc van Gool and Xavier Gir{\'o}-i-Nieto and Jordi Torres} } @conference {cLina, title = {Disentangling Motion, Foreground and Background Features in Videos}, booktitle = {CVPR 2017 Workshop Brave New Motion Representations}, year = {2017}, month = {05/2017}, abstract = {

This paper instroduces an unsupervised framework to extract semantically rich features for video representation. Inspired by how the human visual system groups objects based on motion cues, we propose a deep convolutional neural network that disentangles motion, foreground and background information. The proposed architecture consists of a 3D convolutional feature encoder for blocks of 16 frames, which is trained for reconstruction tasks over the first and last frames of the sequence. The model is trained with a fraction of videos from the UCF-101 dataset taking as ground truth the bounding boxes around the activity regions. Qualitative results indicate that the network can successfully update the foreground appearance based on pure-motion features. The benefits of these learned features are shown in a discriminative classification task when compared with a random initialization of the network weights, providing a gain of accuracy above the 10\%.

Disentangle motion, Foreground and Background Features in Videos from Xavier Giro-i-Nieto
}, author = {Lin, Xunyu and V{\'\i}ctor Campos and Xavier Gir{\'o}-i-Nieto and Jordi Torres and Cristian Canton-Ferrer} } @conference {c, title = {Distributed training strategies for a computer vision deep learning algorithm on a distributed GPU cluster}, booktitle = {International Conference on Computational Science (ICCS)}, year = {2017}, month = {06/2017}, publisher = {Elsevier}, organization = {Elsevier}, address = {Zurich, Switzerland}, abstract = {

Deep learning algorithms base their success on building high learning capacity models with millions of parameters that are tuned in a data-driven fashion. These models are trained by processing millions of examples, so that the development of more accurate algorithms is usually limited by the throughput of the computing devices on which they are trained. In this work, we explore how the training of a state-of-the-art neural network for computer vision can be parallelized on a distributed GPU cluster. The effect of distributing the training process is addressed from two different points of view. First, the scalability of the task and its performance in the distributed setting are analyzed. Second, the impact of distributed training methods on the final accuracy of the models is studied.

[ICCS 2017 website][Related session in ICCS 2017][Paper in UPCommons]

}, keywords = {distributed computing; parallel systems; deep learning; Convolutional Neural Networks}, doi = {https://doi.org/10.1016/j.procs.2017.05.074}, url = {http://www.sciencedirect.com/science/article/pii/S1877050917306129}, author = {V{\'\i}ctor Campos and Sastre, Francesc and Yag{\"u}es, Maurici and M{\'\i}riam Bellver and Xavier Gir{\'o}-i-Nieto and Jordi Torres} }