@conference {cDuartea, title = {Wav2Pix: Speech-conditioned Face Generation using Generative Adversarial Networks}, booktitle = {ICASSP}, year = {2019}, month = {05/2019}, publisher = {IEEE}, organization = {IEEE}, address = {Brighton, UK}, abstract = {

Speech is a rich biometric signal that contains information\ about the identity, gender and emotional state of the speaker.\ In this work, we explore its potential to generate face images\ of a speaker by conditioning a Generative Adversarial\ Network (GAN) with raw speech input. We propose a deep\ neural network that is trained from scratch in an end-to-end\ fashion, generating a face directly from the raw speech waveform\ without any additional identity information (e.g reference\ image or one-hot encoding). Our model is trained in a\ self-supervised fashion by exploiting the audio and visual signals\ naturally aligned in videos. With the purpose of training\ from video data, we present a novel dataset collected for this\ work, with high-quality videos of ten youtubers with notable\ expressiveness in both the speech and visual signals.

}, doi = {10.1109/ICASSP.2019.8682970}, url = {http://hdl.handle.net/2117/167073}, author = {Amanda Duarte and Rold{\'a}n, Francisco and Tubau, Miquel and Escur, Janna and Pascual-deLaPuente, Santiago and Amaia Salvador and Mohedano, Eva and McGuinness, Kevin and Jordi Torres and Xavier Gir{\'o}-i-Nieto} } @mastersthesis {xRoldana, title = {Speech-conditioned Face Generation with Deep Adversarial Networks}, year = {2018}, abstract = {

Image synthesis have been a trending task for the AI community in recent years. Many works have shown the potential of Generative Adversarial Networks (GANs) to deal with tasks such as text or audio to image synthesis. In particular, recent advances in deep learning using audio have inspired many works involving both visual and auditory information. In this work we propose a face synthesis method using audio and/or language representations as inputs. Furthermore, a dataset which relates speech utterances with a face and an identity has been built, fitting for other tasks apart from face synthesis such as speaker recognition or voice conversion.

Speech Conditioned Face Generation with Deep Adversarial Networks from Universitat Polit{\`e}cnica de Catalunya
}, author = {Rold{\'a}n, Francisco}, editor = {Pascual-deLaPuente, Santiago and Amaia Salvador and McGuinness, Kevin and Xavier Gir{\'o}-i-Nieto} } @article {xGiro-i-Nieto17, title = {La meitat de les not{\'\i}cies que consumirem el 2022 seran falses}, year = {2017}, publisher = {Corporaci{\'o} Catalana de Mitjans Audiovisuals}, address = {Sant Joan Desp{\'\i}}, abstract = {

Reportatge em{\`e}s dins el Telenot{\'\i}cies Vespre de Televisi{\'o} de Catalunya el diumenge 26 de novembre de 2017.

Els programes d{\textquoteright}intel{\textperiodcentered}lig{\`e}ncia artificial s{\'o}n capa{\c c}os de crear imatges i veus cada cop m{\'e}s realistes i obren la porta a generar mentides de forma m{\'e}s automatitzada

}, keywords = {deep learning, fake news, gan}, url = {http://www.ccma.cat/324/la-meitat-de-les-noticies-que-consumirem-el-2022-seran-falses/noticia/2823178/}, author = {Xavier Gir{\'o}-i-Nieto and Pascual-deLaPuente, Santiago and Mir{\'o}, Vict{\`o}ria and Esteve, Oriol} } @mastersthesis {xRoldan, title = {Visual Question Answering 2.0}, year = {2017}, abstract = {

This bachelor{\textquoteright}s thesis explores di erent deep learning techniques to solve the Visual Question-Answering (VQA) task, whose aim is to answer questions about images. We study di erent Convolutional\ Neural Networks (CNN) to extract the visual representation from images: Kernelized-CNN (KCNN), VGG-16 and Residual Networks (ResNet). We also analyze the impact of using\ pre-computed word embeddings trained in large datasets (GloVe embeddings). Moreover, we\ examine di erent techniques of joining representations from di erent modalities. This work has\ been submitted to the second edition Visual Question Answering Challenge, and obtained a\ 43.48\% of accuracy.

[Project page]

Visual Question Answering 2.0 from Xavier Giro-i-Nieto
}, author = {Rold{\'a}n, Francisco}, editor = {Xavier Gir{\'o}-i-Nieto and Masuda-Mora, Issey and Pascual-deLaPuente, Santiago} } @mastersthesis {xMasuda-Mora, title = {Open-Ended Visual Question-Answering}, year = {2016}, abstract = {

Advisors: Santiago de la Puente and Xavier Gir{\'o}-i-Nieto

Studies: Bachelor degree in Science and Telecommunication Technologies Engineering at\ Telecom BCN-ETSETB\ from the Technical University of Catalonia (UPC)

Grade: A with honors (10/10.0)

This thesis studies methods to solve Visual Question-Answering (VQA) tasks with a Deep Learning framework.As a preliminary step, we explore Long Short-Term Memory (LSTM) networks used in Natural Language Processing (NLP) to tackle Question-Answering (text based). We then modify the previous model to accept an image as an input in addition to the question. For this purpose, we explore the VGG-16 and K-CNN convolutional neural networks to extract visual features from the image. These are merged with the word embedding or with a sentence embedding of the question to predict the answer. This work was successfully submitted to the Visual Question Answering Challenge 2016, where it achieved a 53,62\% of accuracy in the test dataset. The developed software has followed the best programming practices and Python code style, providing a consistent baseline in Keras for different configurations. The source code and models are publicly available at https://github.com/imatge-upc/vqa-2016-cvprw.

Open-ended Visual Question-Answering from Xavier Giro
}, url = {https://upcommons.upc.edu/handle/2117/89671}, author = {Masuda-Mora, Issey}, editor = {Pascual-deLaPuente, Santiago and Xavier Gir{\'o}-i-Nieto} } @conference {cMontes, title = {Temporal Activity Detection in Untrimmed Videos with Recurrent Neural Networks}, booktitle = {1st NIPS Workshop on Large Scale Computer Vision Systems 2016}, year = {2016}, month = {12/2016}, abstract = {

This work proposes a simple pipeline to classify and temporally localize activities in untrimmed videos. Our system uses features from a 3D Convolutional Neural Network (C3D) as input to train a a recurrent neural network (RNN) that learns to classify video clips of 16 frames. After clip prediction, we post-process the output of the RNN to assign a single activity label to each video, and determine the temporal boundaries of the activity within the video. We show how our system can achieve competitive results in both tasks with a simple architecture. We evaluate our method in the ActivityNet Challenge 2016, achieving a 0.5874 mAP and a 0.2237 mAP in the classification and detection tasks, respectively. Our code and models are publicly available at:\ https://imatge-upc.github.io/activitynet-2016-cvprw/

}, author = {Montes, Alberto and Amaia Salvador and Pascual-deLaPuente, Santiago and Xavier Gir{\'o}-i-Nieto} } @article {xMora, title = {Towards Automatic Generation of Question Answer Pairs from Images}, year = {2016}, institution = {Visual Question Answering Challenge Workshop, CVPR 2016}, address = {Las Vegas, NV, USA}, abstract = {

This extended abstract presents our research in the generic field of Visual Question-Answering (VQA) focusing on a new branch that aims to generate question-answer pairs based on an image.\ To do so, we use the VQA dataset provided for the VQA challenge to train a Deep Neural Network which has the image as an input and two different outputs, the question and its associated answer.

Extended abstract presented as poster in the Visual Question Answering Challenge Workshop, CVPR 2016.

}, url = {http://www.visualqa.org/abstracts.html}, author = {Masuda-Mora, Issey and Pascual-deLaPuente, Santiago and Xavier Gir{\'o}-i-Nieto} }