@conference {cSchrenk24, title = {6G-EWOC: Optical Wireless Communication in Support of Autonomous Driving}, booktitle = {IEEE Future Networks World Forum: Symposium on Future Networks for Connected and Automated Mobility }, year = {2024}, month = {10/2024}, publisher = {IEEE}, organization = {IEEE}, address = {Dubai}, abstract = {

The road towards an efficient transport infrastructure, which safely operates fully autonomous vehicles in congestion-free traffic flows, is subject to several technological challenges that reach also into the realm of telecommunications. Future 6G networks are required to handle huge amounts of sensor information, both in terms of low-latency communication and real-time data processing. The 6G-EWOC project contributes to this challenge through a concerted use of optical wireless communication and sensing with radio-frequency based cellular technology. Three optical-wireless scenarios are discussed, including (i) instant data exchange between vehicles leveraging their lighting assets for low-latency communication, (ii) the provision of vehicular data offloading to a fiber-based 6G infrastructure through beamsteering-assisted optical remote radio heads to enable data fusion at the edge cloud, and (iii) the seamless wireless extension of fiber-grade capacity to isolated network segments situated in fiber-scarce brownfields, which is accomplished through transparent optical air interfaces that ensure fiber-grade continuity by preventing a translucent electro-optic termination of the single-mode fiber channel. Another important contribution aims to advance light- and radio-based sensing technologies towards networked versions that offer simultaneous communication and detecting capabilities.

}, doi = {10.5281/zenodo.13904356}, url = {https://publications.ait.ac.at/en/publications/6g-ewoc-optical-wireless-communication-in-support-of-autonomous-d}, author = {Schrenk, B. and Theodoropoulou, E. and Lyberopoulos, G. and Marcus, C. and Eriksson, O. and Jos{\'e} Antonio L{\'a}zaro and Casas, J. and Federico Dios and Pablo Garc{\'\i}a-G{\'o}mez and S. Royo and Jordi Riu and F{\`a}brega, J.M.} } @article {aGene-Molad, title = {AmodalAppleSize_RGB-D dataset: RGB-D images of apple trees annotated with modal and amodal segmentation masks for fruit detection, visibility and size estimation}, journal = {Data in Brief}, volume = {52}, year = {2024}, month = {02/2024}, abstract = {

The present dataset comprises a collection of RGB-D apple tree images that can be used to train and test computer vision-based fruit detection and sizing methods. This dataset encompasses two distinct sets of data obtained from a Fuji and an Elstar apple orchards. The Fuji apple orchard sub-set consists of 3925 RGB-D images containing a total of 15335 apples annotated with both modal and amodal apple segmentation masks. Modal masks denote the visible portions of the apples, whereas amodal masks encompass both visible and occluded apple regions. Notably, this dataset is the first public resource to incorporate on-tree fruit amodal masks. This pioneering inclusion addresses a critical gap in existing datasets, enabling the development of robust automatic fruit sizing methods and accurate fruit visibility estimation, particularly in the presence of partial occlusions. Besides the fruit segmentation masks, the dataset also includes the fruit size (calliper) ground truth for each annotated apple. The second sub- set comprises 2731 RGB-D images capturing five Elstar apple trees at four distinct growth stages. This sub-set includes mean diameter information for each tree at every growth stage and serves as a valuable resource for evaluating fruit sizing methods trained with the first sub-set. The present data was employed in the research papers titled "Looking behind occlusions: a study on amodal segmentation for robust on-tree apple fruit size estimation" [1] and {\textquotedblleft}Simultaneous fruit detection and size estimation using multitask deep neural networks{\textquotedblright} [2].

}, keywords = {Agricultural robotics, amodal segmentation, depth image, Fruit measurement, Fruit visibility, Instance Segmentation, modal segmentation, Yield prediction}, doi = {https://doi.org/10.1016/j.dib.2023.110000}, author = {Gen{\'e}-Mola, Jordi and Ferrer-Ferrer, M. and Hemming, J. and Dalfsen, P. and Hoog, D. and Sanz-Cortiella, R. and Rosell-Polo, Joan R. and Morros, J.R. and Ver{\'o}nica Vilaplana and Ruiz-Hidalgo, J. and Gregorio, Eduard} } @conference {cPinad, title = {Cell-DETR: Efficient cell detection and classification in WSIs with transformers}, booktitle = {Medical Imaging with Deep Learning (MIDL 2024)}, year = {2024}, month = {07/2024}, abstract = {

Understanding cell interactions and subpopulation distribution is crucial for pathologists to support their diagnoses. This cell information is traditionally extracted from segmentation methods, which poses significant computational challenges on processing Whole Slide Images (WSIs) due to their giga-size nature. Nonetheless, the clinically relevant tasks are nuclei detection and classification rather than segmentation. In this manuscript, we undertake a comprehensive exploration of the applicability of detection transformers for cell detection and classification (Cell-DETR). Not only do we demonstrate the effectiveness of the method by achieving state-of-the-art performance on well-established benchmarks, but we also develop a pipeline to tackle these tasks on WSIs at scale to enable the development of downstream applications. We show its efficiency and feasibility by reporting a significantly faster inference time on a dataset featuring large WSIs. By addressing the challenges associated with large-scale cell detection, our work contributes valuable insights that pave the way for improved diagnostics and downstream applications leveraging cell-level information.

}, author = {Oscar Pina and Eduard Dorca and Ver{\'o}nica Vilaplana} } @article {aDios24, title = {Experimental evaluation of a MIMO radar performance for ADAS application}, journal = {Telecom}, volume = {5}, year = {2024}, month = {06/2024}, pages = {508-521}, chapter = {508}, abstract = {

Among the sensors necessary to equip vehicles with an autonomous driving system, there is a tacit agreement that cameras and some type of radar would be essential. The ability of radar to spatially locate objects (pedestrians, other vehicles, trees, street furniture, and traffic signs) makes it the most economical complement to the cameras in the visible spectrum in order to give the correct depth to scenes. From the echoes obtained by the radar, some data fusion algorithms will try to locate each object in its correct place within the space surrounding the vehicle. In any case, the usefulness of the radar will be determined by several performance parameters, such as its average error in distance, the maximum errors, and the number of echoes per second it can provide. In this work, we have tested experimentally the AWR1843 MIMO radar from Texas Instruments to measure those parameters.

}, keywords = {advanced driver assistance systems, autonomous vehicles, MIMO radar, radar}, doi = {10.3390/telecom5030026}, author = {Federico Dios and Sergio Torres-Benito and Jos{\'e} Antonio L{\'a}zaro and Casas, J. and Jorge Pinazo and Adolfo Ler{\'\i}n} } @inbook {bHernandez, title = {Interpreting Machine Learning Models for Survival Analysis: A Study of Cutaneous Melanoma Using the SEER Database}, booktitle = {Explainable Artificial Intelligence and Process Mining Applications for Healthcare,}, volume = {2020}, number = {Communications in Computer and Information Science}, year = {2024}, publisher = {Springer, Cham}, organization = {Springer, Cham}, abstract = {

In this study, we train and compare three types of machine learning algorithms for Survival Analysis: Random Survival Forest, DeepSurv and DeepHit, using the SEER database to model cutaneous malignant melanoma. Additionally, we employ SurvLIMEpy library, a Python package designed to provide explainability for survival machine learning models, to analyse feature importance. The results demonstrate that machine learning algorithms outperform the Cox Proportional Hazards Model. Our work underscores the importance of explainability methods for interpreting black-box models and provides insights into important features related to melanoma prognosis.

}, isbn = {978-3-031-54302-9}, issn = {978-3-031-54303-6}, doi = {https://doi.org/10.1007/978-3-031-54303-6_6}, author = {Carlos Hernandez and Cristian Pach{\'o}n-Garc{\'\i}a and Pedro Delicado and Ver{\'o}nica Vilaplana} } @article {aPachon-Garcia, title = {SurvLIMEpy: A Python package implementing SurvLIME}, journal = {Expert Systems With Applications}, volume = {237, Part C}, year = {2024}, month = {03/2024}, abstract = {

In this paper we present SurvLIMEpy, an open-source Python package that implements the SurvLIME algorithm. This method allows to compute local feature importance for machine learning algorithms designed for modelling Survival Analysis data. Our\ implementation takes advantage of the parallelisation paradigm as all computations are performed in a matrix-wise fashion which speeds up execution time. Additionally, SurvLIMEpy assists the user with visualization tools to better understand the result of the algorithm. The package supports a wide variety of survival models, from the Cox Proportional Hazards Model to deep learning models such as DeepHit or DeepSurv. Two types of experiments are presented in this paper. First, by means of simulated data, we study the ability of the algorithm to capture the importance of the features. Second, we use three open source survival datasets together with a set of survival algorithms in order to demonstrate how SurvLIMEpy behaves when applied to different models.

}, keywords = {eXplainable Artificial Intelligence, Interpretable Machine Learning, machine learning, python, Survival Analysis}, author = {Cristian Pach{\'o}n-Garc{\'\i}a and Carlos Hernandez and Pedro Delicado and Ver{\'o}nica Vilaplana} } @article {ade-Mas-Gimenez23, title = {Gradient-Based Metrics for the Evaluation of Image Defogging}, journal = {World Electric Vehicle Journal}, volume = {14}, year = {2023}, month = {09/2023}, chapter = {254}, abstract = {

Fog, haze, or smoke are standard atmospheric phenomena that dramatically compromise the overall visibility of any scene, critically affecting features such as the illumination, contrast, and contour detection of objects. The decrease in visibility compromises the performance of computer vision algorithms such as pattern recognition and segmentation, some of which are very relevant to decision-making in the field of autonomous vehicles. Several dehazing methods have been proposed that either need to estimate fog parameters through physical models or are statistically based. But physical parameters greatly depend on the scene conditions, and statistically based methods require large datasets of natural foggy images together with the original images without fog, i.e., the ground truth, for evaluation. Obtaining proper fog-less ground truth images for pixel-to-pixel evaluation is costly and time-consuming, and this fact hinders progress in the field. This paper aims to tackle this issue by proposing gradient-based metrics for image defogging evaluation that do not require a ground truth image without fog or a physical model. A comparison of the proposed metrics with metrics already used in the NTIRE 2018 defogging challenge as well as several state-of-the-art defogging evaluation metrics is performed to prove its effectiveness in a general situation, showing comparable results to conventional metrics and an improvement in the no-reference scene. A Matlab implementation of the proposed metrics has been developed and it is open-sourced in a public GitHub repository.

}, doi = {10.3390/wevj14090254}, url = {https://www.mdpi.com/2032-6653/14/9/254}, author = {Gerard de-Mas-Gim{\'e}nez and Pablo Garc{\'\i}a-G{\'o}mez and Casas, J. and S. Royo} } @conference {cHernandez23, title = {Interpreting Machine Learning models for Survival Analysis: A study of Cutaneous Melanoma using the SEER Database}, booktitle = {XAI-Healthcare 2023 Workshop at 21st International Conference of Artificial Intelligence in Medicine (AIME 2023)}, year = {2023}, month = {06/2023}, address = {Portoroz, Slovenia}, author = {Carlos Hernandez and Cristian Pach{\'o}n-Garc{\'\i}a and Pedro Delicado and Ver{\'o}nica Vilaplana} } @article {aDominguez, title = {Neural ADMIXTURE: rapid population clustering with autoencoders}, journal = {Nature Computational Science}, year = {2023}, month = {07/2023}, abstract = {

Characterizing the genetic substructure of large cohorts has become increasingly important as genetic association and prediction studies are extended to massive, increasingly diverse, biobanks. ADMIXTURE and STRUCTURE are widely used unsupervised clustering algorithms for characterizing such ancestral genetic structure. These methods decompose individual genomes into fractional cluster assignments with each cluster representing a vector of DNA marker frequencies. The assignments, and clusters, provide an interpretable representation for geneticists to describe population substructure at the sample level. However, with the rapidly increasing size of population biobanks and the growing numbers of variants genotyped (or sequenced) per sample, such traditional methods become computationally intractable. Furthermore, multiple runs with different hyperparameters are required to properly depict the population clustering using these traditional methods, increasing the computational burden. This can lead to days of compute. In this work we present Neural ADMIXTURE, a neural network autoencoder that follows the same modeling assumptions as ADMIXTURE, providing similar (or better) clustering, while reducing the compute time by orders of magnitude. Indeed, the equivalent of one month of continuous compute can be reduced to hours. In addition, Neural ADMIXTURE can include multiple outputs, providing the equivalent results as running the original ADMIXTURE algorithm many times with different numbers of clusters. Our models can also be stored, allowing later cluster assignment to be performed with a linear computational time.

}, author = {Dominguez, Albert and Mas-Montserrat, Daniel and Bustamante, Carlos and Xavier Gir{\'o}-i-Nieto and Ioannidis, Alexander G.} } @conference {cLozano23, title = {Optical Phased Array Antenna Apodization for Lidar in Autonomous Vehicles}, booktitle = {XIII Reuni{\'o}n OptoElectr{\'o}nica}, year = {2023}, month = {06/2023}, publisher = {OPTOEL}, organization = {OPTOEL}, address = {Sevilla, Spain}, abstract = {

This paper presents the specific design of an Optical Phased Array antenna (OPA) to apodize the emission of a lidar in the context of a project where diverse optoelectronic sensors such as cameras, radars, and commercial lidars are used to provide data in order to fuse them and develop perception for robots as future autonomous vehicles. While mechanical based lidars are already commercially available, this work focuses on designing much more robust and potentially cheaper lidars based on photonic integrated circuits and energy optimization through the apodization of the emission of the OPA.

}, url = {https://www.optoel2023.es/94299/section/44205/optoel-2023.html}, author = {Jos{\'e} Lozano and Humberto Jim{\'e}nez and Sergio Torres and Pau Biosca and Bernat Fontanet and Jorge Pinazo and Adolfo Ler{\'\i}n and Federico Dios and Casas, J. and Jos{\'e} Antonio L{\'a}zaro} } @article {aPlasencia23, title = {A Preliminary Study of Deep Learning Sensor Fusion for Pedestrian Detection}, journal = {Sensors}, volume = {23}, year = {2023}, month = {04/2023}, chapter = {4167}, abstract = {

Most pedestrian detection methods focus on bounding boxes based on fusing RGB with lidar. These methods do not relate to how the human eye perceives objects in the real world. Furthermore, lidar and vision can have difficulty detecting pedestrians in scattered environments, and radar can be used to overcome this problem. Therefore, the motivation of this work is to explore, as a preliminary step, the feasibility of fusing lidar, radar, and RGB for pedestrian detection that potentially can be used for autonomous driving that uses a fully connected convolutional neural network architecture for multimodal sensors. The core of the network is based on SegNet, a pixel-wise semantic segmentation network. In this context, lidar and radar were incorporated by transforming them from 3D pointclouds into 2D gray images with 16-bit depths, and RGB images were incorporated with three channels. The proposed architecture uses a single SegNet for each sensor reading, and the outputs are then applied to a fully connected neural network to fuse the three modalities of sensors. Afterwards, an up-sampling network is applied to recover the fused data. Additionally, a custom dataset of 60 images was proposed for training the architecture, with an additional 10 for evaluation and 10 for testing, giving a total of 80 images. The experiment results show a training mean pixel accuracy of 99.7\% and a training mean intersection over union of 99.5\%. Also, the testing mean of the IoU was 94.4\%, and the testing pixel accuracy was 96.2\%. These metric results have successfully demonstrated the effectiveness of using semantic segmentation for pedestrian detection under the modalities of three sensors. Despite some overfitting in the model during experimentation, it performed well in detecting people in test mode. Therefore, it is worth emphasizing that the focus of this work is to show that this method is feasible to be used, as it works regardless of the size of the dataset. Also, a bigger dataset would be necessary to achieve a more appropiate training. This method gives the advantage of detecting pedestrians as the human eye does, thereby resulting in less ambiguity. Additionally, this work has also proposed an extrinsic calibration matrix method for sensor alignment between radar and lidar based on singular value decomposition.

}, keywords = {autonomous driving, Convolutional Neural Networks, sensor calibration, sensor fusion}, doi = {10.3390/s23084167}, url = {https://www.mdpi.com/1424-8220/23/8/4167}, author = {Alfredo Ch{\'a}vez Plasencia and Pablo Garc{\'\i}a-G{\'o}mez and Eduardo Bernal P{\'e}rez and Gerard de-Mas-Gim{\'e}nez and Casas, J. and S. Royo} } @conference {cMorros, title = {Real-time lane classification and accident detection for safer micromobility}, booktitle = {11th Internationa Congress on Transportation Research}, year = {2023}, month = {09/2023}, address = {Heraklion, Crete}, abstract = {

The lack of knowledge of the micromobility regulations by e-scooter users is an important factor behind some of\ the accidents involving these vehicles. We present two modules that can increase the safety of the users and\ pedestrians: First, a computer vision model that analyses the video feed captured with a smartphone attached to\ the e-scooter, and predicts in real-time the type of lane in which the user is riding. This knowledge is used by an\ application which combines this information with GSNN location information and a database of mobility\ regulations, and informs the user when he/she is not complying with these regulations. Second, an accident\ detection system, using the smartphone accelerometer, that detects if there is a fall during the riding, so that the\ app can contact the authorities to determine the appropriate response. The experimental results show excellent\ results for both modules.

}, keywords = {accident detection, computer Vision, deep learning, lane classification, Micromobility, RideSafeUM}, author = {Morros, J.R. and Broquetas, A. and Mateo, A. and Puig, J. and Davins, M.} } @conference {cTarresa, title = {Sign Language Translation from Instructional Videos}, booktitle = {CVPR 2023 Women in Computer Vision Workshop}, year = {2023}, month = {04/2023}, publisher = {Computer Vision Foundation / IEEE}, organization = {Computer Vision Foundation / IEEE}, address = {Vancouver, Canada}, abstract = {

The advances in automatic sign language translation (SLT) to spoken languages have been mostly benchmarked with datasets of limited size and restricted domains. Our work advances the state of the art by providing the first baseline results on How2Sign, a large and broad dataset. We train a Transformer over I3D video features, using the reduced BLEU as a reference metric for validation, instead of the widely used BLEU score. We report a result of 8.03 on the BLEU score, and publish the first open-source implementation of its kind to promote further advances.

}, author = {Laia Tarr{\'e}s and Gerard I. G{\'a}llego and Amanda Duarte and Jordi Torres and Xavier Gir{\'o}-i-Nieto} } @conference {cde-Mas-Gimenez23, title = {Single image defogging and evaluation with deep neural networks without the need of ground truth}, booktitle = {SPIE Automated Visual Inspection and Machine Vision V}, year = {2023}, month = {05/2023}, publisher = {Proc. SPIE 12623}, organization = {Proc. SPIE 12623}, address = {Munich}, abstract = {

Fog dramatically compromises the overall visibility of any scene, critically affecting features such as objects{\textquoteright} illumination, contrast, and contours. The decrease in visibility compromises the performance of Computer Vision algorithms such as pattern recognition and segmentation, some of them very relevant to decision-making in the rise of autonomous-driven vehicles. Many dehazing methods have been proposed. However, to the best of our knowledge, all currently used metrics do compare the defogged image to its ground truth, usually the same scene on a non-foggy day, or estimate physical parameters from the scene. This hinders progress in the field, as obtaining proper ground truth images is not always possible and becomes costly and time-consuming because physical parameters greatly depend on the scene conditions. This work aims to tackle this issue by proposing a real-time operating defogging network that only takes an RGB image of the fogged scene as input, performs the defogging, and uses a contour-based metric for Single Image Defogging evaluation even when the ground truth is not available, which is the most common situation. The proposed metric only requires the original hazy image and the image after the defogging procedure. We trained our network using a novel two-stage pipeline with the DENSE dataset and compared our method and metric with currently used metrics and other defogging techniques with the NTIRE 2018 defogging challenge to prove their effectiveness.

}, doi = {10.1117/12.2673442}, author = {Gerard de-Mas-Gim{\'e}nez and Pablo Garc{\'\i}a-G{\'o}mez and Casas, J. and Santiago Royo} } @phdthesis {dDuarte, title = {Data and methods for a visual understanding of sign languages}, volume = {PhD}, year = {2022}, type = {International Mention}, abstract = {

Signed languages are complete and natural languages used as the first or preferred mode of communication by millions of people worldwide. However, they, unfortunately, continue to be marginalized languages. Designing, building, and evaluating models that work on sign languages presents compelling research challenges and requires interdisciplinary and collaborative efforts. The recent advances in Machine Learning (ML) and Artificial Intelligence (AI) has the power to enable better accessibility to sign language users and narrow down the existing communication barrier between the Deaf community and non-sign language users. However, recent AI-powered technologies still do not account for sign language in their pipelines. This is mainly because sign languages are visual languages, that use manual and non-manual features to convey information, and do not have a standard written form. Thus, the goal of this thesis is to contribute to the development of new technologies that account for sign language by creating large-scale multimodal resources suitable for training modern data-hungry machine learning models and developing automatic systems that focus on computer vision tasks related to sign language that aims at learning better visual understanding of sign languages.Thus, in Part I, we introduce the How2Sign dataset, which is a large-scale collection of multimodal and multiview sign language videos in American Sign Language. In Part II, we contribute to the development of technologies that account for sign languages by presenting in Chapter 4 a framework called Spot-Align, based on sign spotting methods, to automatically annotate sign instances in continuous sign language. We further present the benefits of this framework and establish a baseline for the sign language recognition task on the How2Sign dataset. In addition to that, in Chapter 5 we benefit from the different annotations and modalities of the How2Sign to explore sign language video retrieval by learning cross-modal embeddings. Later in Chapter 6, we explore sign language video generation by applying Generative Adversarial Networks to the sign language domain and assess if and how well sign language users can understand automatically generated sign language videos by proposing an evaluation protocol based on How2Sign topics and English translation.

}, author = {Amanda Duarte}, editor = {Jordi Torres and Xavier Gir{\'o}-i-Nieto} } @article {aTemprana-Salvador22, title = {DigiPatICS: Digital Pathology Transformation of the Catalan Health Institute Network of 8 Hospitals - Planification, Implementation and Preliminary Results}, journal = {Diagnostics}, volume = {12}, year = {2022}, month = {03/2022}, chapter = {852}, abstract = {

Complete digital pathology transformation for primary histopathological diagnosis is a challenging yet rewarding endeavor. Its advantages are clear with more efficient workflows, but there are many technical and functional difficulties to be faced. The Catalan Health Institute (ICS) has started its DigiPatICS project, aiming to deploy digital pathology in an integrative, holistic, and comprehensive way within a network of 8 hospitals, over 168 pathologists, and over 1 million slides each year. We describe the bidding process and the careful planning that was required, followed by swift implementation in stages. The purpose of the DigiPatICS project is to increase patient safety and quality of care, improving diagnosis and the efficiency of processes in the pathological anatomy departments of the ICS through process improvement, digital pathology, and artificial intelligence tools.

}, keywords = {artificial intelligence, computational pathology, deep learning, digital pathology, implementation, LIS, primary diagnosis, telepathology, workflow}, doi = {10.3390/diagnostics12040852}, url = {https://www.mdpi.com/2075-4418/12/4/852}, author = {Jordi Temprana-Salvador and Pau L{\'o}pez-Garc{\'\i}a and Josep Castellv{\'\i} Vives and Llu{\'\i}s de Haro and Eudald Ballesta and Matias Rojas Abusleme and Miquel Arrufat and Ferran Marques and Casas, J. and Carlos Gallego and Laura Pons and Jos{\'e} Luis Mate and Pedro Luis Fern{\'a}ndez and Eugeni L{\'o}pez-Bonet and Ramon Bosch and Salom{\'e} Mart{\'\i}nez and Santiago Ram{\'o}n y Cajal and Xavier Matias-Guiu} } @mastersthesis {xDomenech, title = {Hiding Images in their Spoken Narratives}, year = {2022}, abstract = {

Steganography is the technique of hiding secret data within an ordinary, non-secret, file or message in order to avoid its detection. Throughout our work, we study the case where the hidden secret data is an image and the non-secret data or cover signal is an audio. To this end, we use a recently proposed residual architecture operating on top of short-time discrete cosine transform (STDCT) audio spectrograms. In our work, we evaluate the above mentioned residual steganography architecture with the Localized Narratives dataset, explore the feasibility of using short-time fourier transform (STFT) audio spectrograms instead of STDCTs to improve the efficiency of the system, investigate the use of hidden signals permuted with the objective to spread the audio corruption of the revealed images, apply averaged audio windows to improve quality results and tested the system in real-world distortions.

}, author = {Teresa Domenech}, editor = {McGuinness, Kevin and Pons, Jordi and Xavier Gir{\'o}-i-Nieto} } @article {pTerradas22, title = {A method, system and computer programs to automatically transform an image}, journal = {European Patent Office}, volume = {21382176}, year = {2022}, abstract = {

The present invention is directed, in general, to a method and a system to automatically transform an image using neural networks. More specifically, the invention relates to a controllable image generation through an image representation and several conditions using a conditional Neural Network.

The method comprises receiving, by a processing unit, at least one image and processing the received image to obtain an image representation thereof (i.e. an intermediate representation of the initial image that captures high level features and low level properties of the image and that is structured in an understandable way for a conditional Neural Network such as a deep generative Neural Network). The method also includes receiving, by an encoding unit, one or more references (e.g. other images, text, labels, combinations thereof, or even other data describing how the received image should be transformed) and encoding the received one or more references into one or more features, the latter being further provided to a conditional Neural Network as a condition(s). In addition, the method further applies the conditional Neural Network to transform the obtained image representation into a resulting conditioned image based on said condition(s).

}, issn = {21382176.2}, url = {https://patentscope.wipo.int/search/es/detail.jsf?docId=EP373278976}, author = {Terradas, R. and Pau Domingo and Grau, M. and Alarc{\'o}n, E. and Ruiz-Hidalgo, J.} } @conference {cPinazo22, title = {Perception in the era of Autonomous Vehicles}, booktitle = {Photonics 4 Smart Cities, SCEWC 2022}, year = {2022}, month = {11/2022}, publisher = {Photonics21}, organization = {Photonics21}, address = {Barcelona}, url = {https://www.fotonica21.org/photonics-4-smart-cities}, author = {Jorge Pinazo and Adolfo Ler{\'\i}n and Francesc Xavier de Gibert and {\'A}lvaro Moliner and Daniel Sevilla and Antonio Jurado and Iv{\'a}n R{\'\i}os and Rodrigo Jerez and Jaime Santiago and {\'A}lvaro Linuesa and Antonio Cano and Federico Dios and Adolfo Comer{\'o}n and Casas, J. and Jos{\'e} Antonio L{\'a}zaro} } @article {aa, title = {QU-BraTS: MICCAI BraTS 2020 Challenge on Quantifying Uncertainty in Brain Tumor Segmentation--Analysis of Ranking Metrics and Benchmarking Results}, journal = {Journal of Machine Learning for Biomedical Imaging}, year = {2022}, month = {08/2022}, abstract = {

Deep learning (DL) models have provided state-of-the-art performance in various medical imaging benchmarking challenges, including the Brain Tumor Segmentation (BraTS) challenges. However, the task of focal pathology multi-compartment segmentation (e.g., tumor and lesion sub-regions) is particularly challenging, and potential errors hinder translating DL models into clinical workflows. Quantifying the reliability of DL model predictions in the form of uncertainties could enable clinical review of the most uncertain regions, thereby building trust and paving the way toward clinical translation. Several uncertainty estimation methods have recently been introduced for DL medical image segmentation tasks. Developing scores to evaluate and compare the performance of uncertainty measures will assist the end-user in making more informed decisions. In this study, we explore and evaluate a score developed during the BraTS 2019 and BraTS 2020 task on uncertainty quantification (QU-BraTS) and designed to assess and rank uncertainty estimates for brain tumor multi-compartment segmentation. This score (1) rewards uncertainty estimates that produce high confidence in correct assertions and those that assign low confidence levels at incorrect assertions, and (2) penalizes uncertainty measures that lead to a higher percentage of under-confident correct assertions. We further benchmark the segmentation uncertainties generated by 14 independent participating teams of QU-BraTS 2020, all of which also participated in the main BraTS segmentation task. Overall, our findings confirm the importance and complementary value that uncertainty estimates provide to segmentation algorithms, highlighting the need for uncertainty quantification in medical image analyses. Finally, in favor of transparency and reproducibility, our evaluation code is made publicly available at:\ this https URL.

}, url = {https://www.melba-journal.org/papers/2022:026.html}, author = {Raghav Metha and Angelos Filos and Ujjwal Baid and Laura Mora and Ver{\'o}nica Vilaplana and Christos Davatzikos and Bjoern Menze and Spyridon Bakas and Yarin Gal and Tar Arbel} } @conference {cDuartec, title = {Sign Language Video Retrieval with Free-Form Textual Queries}, booktitle = {CVPR 2022 - CVF/IEEE Conference on Computer Vision and Pattern Recognition}, year = {2022}, abstract = {

Systems that can efficiently search collections of sign language videos have been highlighted as a useful application of sign language technology. However, the problem of searching videos beyond individual keywords has received limited attention in the literature. To address this gap, in this work we introduce the task of sign language retrieval with free-form textual queries: given a written query (e.g., a sentence) and a large collection of sign language videos, the objective is to find the signing video in the collection that best matches the written query. We propose to tackle this task by learning cross-modal embeddings on the recently introduced large-scale How2Sign dataset of American Sign Language (ASL). We identify that a key bottleneck in the performance of the system is the quality of the sign video embedding which suffers from a scarcity of labeled training data. We, therefore, propose SPOT-ALIGN, a framework for interleaving iterative rounds of sign spotting and feature alignment to expand the scope and scale of available training data. We validate the effectiveness of SPOT-ALIGN for learning a robust sign video embedding through improvements in both sign recognition and the proposed video retrieval task.

}, author = {Amanda Duarte and Samuel Albanie and Xavier Gir{\'o}-i-Nieto and G{\"u}l Varol} } @article {xGiro-i-Nieto22, title = {Towards Sign Language Translation and Production}, year = {2022}, abstract = {

Machine translation and computer vision have greatly benefited from the advances in deep learning. A large and diverse amount of textual and visual data have been used to train neural networks whether in a supervised or self-supervised manner. Nevertheless, the convergence of the two fields in sign language translation and production still poses multiple open challenges, like the low video resources, limitations in hand pose estimation, or 3D spatial grounding from poses.

}, author = {Xavier Gir{\'o}-i-Nieto and Amanda Duarte} } @article {xAguilar21, title = {2D-to-3D Lifting of Sign Language Body Poses with Recurrent Neural Networks}, year = {2021}, month = {01/2021}, institution = {UPC ETSETB TelecomBCN}, address = {Barcelona}, abstract = {

This paper aims at improving the quality of a dataset\ that contains multiple sequences of 3D poses extracted from\ American Sign Language videos. Each pose consists of 147\ points with three coordinates each. We propose an algorithm able to correct missing points as well as to add some\ constraints such as the length of the bones. To prove the\ quality of the algorithm{\textquoteright}s outcome, we evaluate the task of\ lifting 2D to 3D poses with a deep learning model trained\ on raw data, and another one trained with the preprocessed\ data.

}, author = {Aguilar, Jordi}, editor = {Xavier Gir{\'o}-i-Nieto and Amanda Duarte} } @conference {cDuarte20, title = {How2Sign: A Large-scale Multimodal Dataset for Continuous American Sign Language}, booktitle = {CVPR 2021}, year = {2021}, month = {06/2021}, abstract = {

Sign Language is the primary means of communication for the majority of the Deaf community. One of the factors that has hindered the progress in the areas of automatic sign language recognition, generation, and translation is the absence of large annotated datasets, especially continuous sign language datasets, i.e. datasets that are annotated and segmented at the sentence or utterance level. Towards this end, in this work we introduce How2Sign, a work-in-progress dataset collection. How2Sign consists of a parallel corpus of 80 hours of sign language videos (collected with multi-view RGB and depth sensor data) with corresponding speech transcriptions and gloss annotations. In addition, a three-hour subset was further recorded in a geodesic dome setup using hundreds of cameras and sensors, which enables detailed 3D reconstruction and pose estimation and paves the way for vision systems to understand the 3D geometry of sign language.





}, author = {Amanda Duarte and S. Palaskar and Lucas Ventura and Ghadiyaram, Deepti and DeHaan, Kenneth and F. Metze and Jordi Torres and Xavier Gir{\'o}-i-Nieto} } @conference {cDominguez, title = {Neural ADMIXTURE: rapid population clustering with autoencoders}, booktitle = {2021 Society for Molecular Biology \& Evolution meeting (SMBEv2021)}, year = {2021}, month = {06/2021}, address = {Virtual}, abstract = {

Characterizing the genetic substructure of large cohorts has become increasingly important as genetic association and prediction studies are extended to massive, increasingly diverse, biobanks. ADMIXTURE and STRUCTURE are widely used unsupervised clustering algorithms for characterizing such ancestral genetic structure. These methods decompose individual genomes into fractional cluster assignments with each cluster representing a vector of DNA marker frequencies. The assignments, and clusters, provide an interpretable representation for geneticists to describe population substructure at the sample level. However, with the rapidly increasing size of population biobanks and the growing numbers of variants genotyped (or sequenced) per sample, such traditional methods become computationally intractable. Furthermore, multiple runs with different hyperparameters are required to properly depict the population clustering using these traditional methods, increasing the computational burden. This can lead to days of compute. In this work we present Neural ADMIXTURE, a neural network autoencoder that follows the same modeling assumptions as ADMIXTURE, providing similar (or better) clustering, while reducing the compute time by orders of magnitude. In addition, this network can include multiple outputs, providing the equivalent results as running the original ADMIXTURE algorithm many times with different numbers of clusters. These models can also be stored, allowing later cluster assignment to be performed with a linear computational time.

}, author = {Dominguez, Albert and Mas-Montserrat, Daniel and Bustamante, Carlos and Xavier Gir{\'o}-i-Nieto and Ioannidis, Alexander G.} } @article {aPuig-Sitjes21, title = {Real-time detection of overloads on the plasma-facing components of Wendelstein 7-X}, journal = {Applied sciences (Basel)}, volume = {11}, year = {2021}, month = {12/2021}, chapter = {1}, issn = {2076-3417}, doi = {10.3390/app112411969}, url = {http://hdl.handle.net/2117/361558}, author = {Puig-Sitjes, A. and Jakubowski, M. and Naujoks, D. and Gao, Y. and Drewelow, P. and Niemann, H. and Felinger, J. and Casas, J. and Salembier, P. and Clemente, R.} } @conference {cVentura20, title = {Can Everybody Sign Now? Exploring Sign Language Video Generation from 2D Poses}, booktitle = {ECCV 2020 Workshop on Sign Language recognition, Production and Translation (SLRTP)}, year = {2020}, month = {08/2020}, abstract = {

Recent work have addressed the generation of human poses represented by 2D/3D coordinates of human joints for sign language. We use the state of the art in Deep Learning for motion transfer and evaluate them on How2Sign, an American Sign Language dataset, to generate videos of signers performing sign language given a 2D pose skeleton. We evaluate the generated videos quantitatively and qualitatively showing that the current models are not enough to generated adequate videos for Sign Language due to lack of detail in hands.

}, author = {Lucas Ventura and Amanda Duarte and Xavier Gir{\'o}-i-Nieto} } @article {9103248, title = {Grounded Sequence to Sequence Transduction}, journal = {IEEE Journal of Selected Topics in Signal Processing}, volume = {14}, year = {2020}, month = {05/2020}, pages = {577-591}, abstract = {

Speech recognition and machine translation have made major progress over the past decades, providing practical systems to map one language sequence to another. Although multiple modalities such as sound and video are becoming increasingly available, the state-of-the-art systems are inherently unimodal, in the sense that they take a single modality {\textemdash} either speech or text {\textemdash} as input. Evidence from human learning suggests that additional modalities can provide disambiguating signals crucial for many language tasks. In this article, we describe the How2 dataset\ , a large, open-domain collection of videos with transcriptions and their translations. We then show how this single dataset can be used to develop systems for a variety of language tasks and present a number of models meant as starting points. Across tasks, we find that building multimodal architectures that perform better than their unimodal counterpart remains a challenge. This leaves plenty of room for the exploration of more advanced solutions that fully exploit the multimodal nature of the How2 dataset\ , and the general direction of multimodal learning with other datasets as well.

}, author = {L. Specia and L. Barrault and O. Caglayan and Amanda Duarte and D. Elliott and S. Gella and N. Holzenberger and C. Lala and S. J. Lee and J. Libovicky and P. Madhyastha and F. Metze and K. Mulligan and A. Ostapenko and S. Palaskar and R. Sanabria and J. Wang and R. Arora} } @mastersthesis {xMuschik20, title = {Learn2Sign : sign language recognition and translation using human keypoint estimation and transformer model}, year = {2020}, abstract = {

Sign language recognition and translation has been an active research field in the recent years with most approaches using deep neural networks to extract information from sign language data. This work investigates the mostly disregarded approach of using human keypoint estimation from image and video data with OpenPose in combination with transformer network architecture. Firstly, it was shown that it is possible to recognize individual signs (4.5\% word error rate (WER)). Continuous sign language recognition though was more error prone (77.3\% WER) and sign language translation was not possible using the proposed methods, which might be due to low accuracy scores of human keypoint estimation by OpenPose and accompanying loss of information or insufficient capacities of the used transformer model. Results may improve with the use of datasets containing higher repetition rates of individual signs or focusing more precisely on keypoint extraction of hands.

}, doi = {10.18419/opus-11197}, url = {https://elib.uni-stuttgart.de/handle/11682/11214}, author = {Muschik, Peter}, editor = {A. Duarte and Xavier Gir{\'o}-i-Nieto} } @conference {cPuig-Sitjes20, title = {Strategy for the real-time detection of thermal events on the plasma facing components of Wendelstein 7-X}, booktitle = {31st Symposium on Fusion Technology (SOFT2020)}, year = {2020}, month = {09/2020}, address = {Dubrovnik, Croatia}, abstract = {

Wendelstein 7-X (W7-X), the most advanced fusion experiment in the stellarator line, aims at demonstrating the feasibility of the stellarator concept as a future fusion power plant. It is planned to restart operation by the end of 2021 with a high heat flux divertor and water-cooled plasma facing components (PFCs) to demonstrate steady-state operation. With plasma energy limits starting at 1 GJ and gradually increasing to 18 GJ over several experimental campaigns, the PFCs have to be protected from overheating. For that, a fully autonomous system is required in order to prevent damage to the plasma facing components due to thermal events.
During the last experimental campaign, when W7-X was equipped with inertially cooled test divertor units, extensive experience was gained with the preliminary design of the thermal event detection system. By then, the system was not yet real-time capable and it was not fully automated, requiring manual supervision between discharges. This experience, however, allowed to prove the validity of some design concepts and to define the new strategy towards the protection of the machine in steady-state operation, when the system will be connected to the Interlock System and the feedback control.
In this work, the design of the real-time thermal event detection system for W7-X for steady-state operation is presented. The system is based on the thermography and video diagnostics to monitor the divertor units, the baffles, and the wall heat-shields and panels. It will be implemented on a real-time system and integrated in CoDaC{\textquoteright}s safety infrastructure. The system relies on computer vision and machine learning techniques to perform a spatio-temporal analysis to detect and classify the thermal events and to perform a risk evaluation. The results and the main conclusions drawn from the analysis of the data from the past campaign are reported.

}, url = {http://hdl.handle.net/21.11116/0000-0007-7FF5-7}, author = {Puig-Sitjes, A. and Jakubowski, M. and Fellinger, J. and Drewelow, P. and Gao, Y. and Niemann, H. and Sunn-Pedersen, T. and K{\"o}nig, R. and Naujoks, D. and Winter, A. and Laqua, H. and Dumke, S. and Moncada, V. and Belafdil, C. and Mitteau, R. and Aumeunier, M.-H. and Pisano, F. and Aymerich, E. and Cannas, B. and Kocsis, G. and Szepesi, T. and Cseh, G. and Szabolics, T. and Casas, J. and Morros, J.R. and Salembier, P. and Clemente, R. and Cobos, M. and I. Caminal and Palacios Corral, A. and Moreno Punzano, A. and Quiceno Lopera, S.} } @conference {cBernal, title = {Transcription-Enriched Joint Embeddings or Spoken Descriptions of Images and Videos}, booktitle = {CVPR 2020 Workshop on Egocentric Perception, Interaction and Computing}, year = {2020}, month = {06/2020}, publisher = {arXiv}, organization = {arXiv}, address = {Seattle, WA, USA}, abstract = {

In this work, we propose an effective approach for training unique embedding representations by combining three simultaneous modalities: image and spoken and textual narratives. The proposed methodology departs from a baseline system that spawns a embedding space trained with only spoken narratives and image cues. Our experiments on the EPIC-Kitchen and Places Audio Caption datasets show that introducing the human-generated textual transcriptions of the spoken narratives helps to the training procedure yielding to get better embedding representations. The triad speech, image and words allows for a better estimate of the point embedding and show an improving of the performance within tasks like image and speech retrieval,\ even when text third modality, text, is not present in the task.

Transcription-Enriched Joint Embeddings for Spoken Descriptions of Images and Videos from Universitat Polit{\`e}cnica de Catalunya
}, author = {Oriol, Benet and Luque, J. and Diego, Ferran and Xavier Gir{\'o}-i-Nieto} } @article {aWang19, title = {Benchmark on Automatic 6-month-old Infant Brain Segmentation Algorithms: The iSeg-2017 Challenge}, journal = {IEEE Transactions on Medical Imaging}, year = {2019}, month = {2019/2/27}, abstract = {

Accurate segmentation of infant brain magnetic resonance (MR) images into white matter (WM), gray matter (GM), and cerebrospinal fluid (CSF) is an indispensable foundation for early studying of brain growth patterns and morphological changes in neurodevelopmental disorders. Nevertheless, in the isointense phase (approximately 6-9 months of age), due to inherent myelination and maturation process, WM and GM exhibit similar levels of intensity in both T1-weighted (T1w) and T2-weighted (T2w) MR images, making tissue segmentation very challenging. Despite many efforts were devoted to brain segmentation, only few studies have focused on the segmentation of 6-month infant brain images. With the idea of boosting methodological development in the community, iSeg-2017 challenge (http://iseg2017.web.unc.edu) provides a set of 6-month infant subjects with manual labels for training and testing the participating methods. Among the 21 automatic segmentation methods participating in iSeg-2017, we review the 8 top-ranked teams, in terms of Dice ratio, modified Hausdorff distance and average surface distance, and introduce their pipelines, implementations, as well as source codes. We further discuss limitations and possible future directions. We hope the dataset in iSeg-2017 and this review article could provide insights into methodological development for the community.

}, doi = {10.1109/TMI.2019.2901712}, author = {Li Wang and Dong Nie and Guannan Li and Elodie Puybareau and Jose Dolz and Qian Zhang and Fan Wang and Jing Xia and Zhengwang Wu and Jiawei Chen and Kim-HanThung and Toan Duc Bui and Jitae Shin and Guodong Zeng and Guoyan Zheng and Vladimir S. Fonov and Andrew Doyle and Yongchao Xu and Pim Moeskops and Josien Pluim and Christian Desrosiers and Ismail Ben Ayed and Gerard Sanroma and Oualid Benkarim and Adri{\`a} Casamitjana and Ver{\'o}nica Vilaplana and Weili Lin and Gang Li and Dinggang Shen} } @mastersthesis {xDomenech19, title = {Clasificaci{\'o}n de im{\'a}genes dermatosc{\'o}picas utilizando Redes Neuronales Convolucionales e informaci{\'o}n de metadatos}, year = {2019}, abstract = {

Convolutional Neural Networks (CNNs) is a technology that is evolving very quickly. The CNNs are used in image recognition to improve computer vision systems in image classification, object detection and segmentation tasks. For all the mentioned features we can apply the CNN in the field of medicine to automate the techniques of obtaining diagnosis and facilitate the tasks of doctors.

The purpose of this project is to classify images of different skin lesions in order to detect possible diseases such as skin cancer (melanoma) through CNNs and metadata information. The images used in this project have been provided by the Dermatology Department of the Clinical Hospital of Barcelona

}, author = {Teresa Domenech}, editor = {Ver{\'o}nica Vilaplana} } @conference {cDuarteb, title = {Cross-modal Neural Sign Language Translation}, booktitle = {Proceedings of the 27th ACM International Conference on Multimedia - Doctoral Symposium}, year = {2019}, month = {10/2019}, publisher = {ACM}, organization = {ACM}, address = {Nice, France}, abstract = {

Sign Language is the primary means of communication for the majority of the Deaf and hard-of-hearing communities. Current computational approaches in this general research area have focused specifically on sign language recognition and the translation of sign language to text. However, the reverse problem of translating from spoken to sign language has so far not been widely explored.

The goal of this doctoral research is to explore sign language translation in this generalized setting, i.e. translating from spoken language to sign language and vice versa. Towards that end, we propose a concrete methodology for tackling the problem of speech to sign language translation and introduce How2Sign, the first public, continuous American Sign Language dataset that enables such research. With a parallel corpus of almost 60 hours of sign language videos (collected with both RGB and depth sensor data) and the corresponding speech transcripts for over 2500 instructional videos, How2Sign is a public dataset of unprecedented scale that can be used to advance not only sign language translation, but also a wide range of sign language understanding tasks.

Poster at ACM Multimedia 2019

Xavier Giro and Amanda Duarte in ACM Multimedia 2019.

}, doi = {10.1145/3343031.3352587}, url = {https://dl.acm.org/citation.cfm?id=3352587}, author = {Amanda Duarte}, editor = {Jordi Torres and Xavier Gir{\'o}-i-Nieto} } @article {aRoisman18, title = {Differential expression of long non-coding RNAs related to proliferation and histological diversity in follicular lymphomas}, journal = {British Journal of Haematology}, volume = {184}, year = {2019}, month = {Feb 2019}, pages = {373-383}, issn = {ISSN:1365-2141}, doi = {DOI: 10.1111/bjh.15656}, author = {A. Roisman and A. Navarro and G. Clot and G. Castellano and B. Gonzalez-Farre and P. P{\'e}rez-Galan and A. Esteve and M. Dabad and S. Heath and M. Gut and Bosio, M. and P. Bellot and Salembier, P. and Albert Oliveras and I. Slavusky and L. Magnano and H. Horn and A. Rosenwald and G. Ott and M. Aymerich and A. L{\'o}pez-Guillermo and P. Jares and J.I. Mart{\'\i}n-Subero and E. Campo and L. Hern{\'a}ndez} } @mastersthesis {xDomingo19, title = {Interpretability of Deep Learning Models}, year = {2019}, abstract = {

In recent years we have seen growth on interest for Deep Learning (DL) algorithms on a variety of problems, due to their outstanding performance. This is more palpable on a multitude of elds, where self learning algorithms are becoming indispensable tools to help professionals solve complex problems.

However as these models are getting better, they also tend to be more complex and are sometimes referred to as "Black Boxes". The lack of explanations for the resulting predictions and the inability of humans to understand those decisions seems problematic.

In this project, dierent methods to increase the interpretability of Deep Neural Networks (DNN) such as Convolutional Neural Network (CNN) are studied. Additionally, we evaluate how these interpretability methods or techniques can be implemented, measured and applied to real-world problems, by creating a python ToolBox.

}, author = {Pau Domingo}, editor = {Ver{\'o}nica Vilaplana} } @conference {cSalvadorf, title = {Inverse Cooking: Recipe Generation from Food Images}, booktitle = {CVPR}, year = {2019}, month = {06/2019}, publisher = {OpenCVF / IEEE}, organization = {OpenCVF / IEEE}, address = {Long Beach, CA, USA}, abstract = {

People enjoy food photography because they appreciate food. Behind each meal there is a story described in a complex recipe and, unfortunately, by simply looking at a food image we do not have access to its preparation process. Therefore, in this paper we introduce an inverse cooking system that recreates cooking recipes given food images. Our system predicts ingredients as sets by means of a novel architecture, modeling their dependencies without imposing any order, and then generates cooking instructions by attending to both image and its inferred ingredients simultaneously. We extensively evaluate the whole system on the large-scale Recipe1M dataset and show that (1) we improve performance w.r.t. previous baselines for ingredient prediction; (2) we are able to obtain high quality recipes by leveraging both image and ingredients; (3) our system is able to produce more compelling recipes than retrieval-based approaches according to human judgment.

}, url = {http://openaccess.thecvf.com/content_CVPR_2019/html/Salvador_Inverse_Cooking_Recipe_Generation_From_Food_Images_CVPR_2019_paper.html}, author = {Amaia Salvador and Drozdzal, Michal and Xavier Gir{\'o}-i-Nieto and Romero, Adriana} } @article {aKuijf, title = {Standardized Assessment of Automatic Segmentation of White Matter Hyperintensities; Results of the WMH Segmentation Challenge}, journal = {IEEE Transactions on Medical Imaging}, year = {2019}, month = {03/2019}, abstract = {

Quantification of white matter hyperintensities (WMH) of presumed vascular origin is of key\ importance in many neurological research studies. Advanced\ measurements are obtained from manual segmentations on brain\ MR images, which is a laborious procedure. Automatic WMH\ segmentation methods exist, but a standardized comparison of\ such methods is lacking. We organized a scientific challenge, in\ which developers could evaluate their method on a standardized\ multi-center/-scanner image dataset, giving an objective comparison:\ the WMH Segmentation Challenge (http://wmh.isi.uu.nl/).\ Sixty T1+FLAIR images from three MR scanners were released\ with manual WMH segmentations. A secret test set of 110\ images from five MR scanners was used for evaluation. Methods\ had to be containerized and submitted to the challenge organizers.Five evaluation metrics were used to rank the methods:\ (1) Dice Similarity Coefficient, (2) modified Hausdorff distance\ (95th percentile), (3) absolute percentage volume difference, (4)\ sensitivity for detecting individual lesions, and (5) F1-score for\ individual lesions. Additionally, methods were ranked on their\ inter-scanner robustness.\ Twenty participants submitted their method for evaluation.\ This paper provides a detailed analysis of the results. In brief,there is a cluster of four methods that rank significantly better\ than the other methods. There is one clear winner, which also\ has the best inter-scanner robustness.\ The challenge remains open for future submissions and provides\ a public platform for method evaluation.

}, keywords = {brain, Evaluation and performance, Magnetic resonance imaging (MRI), segmentation}, issn = {0278-0062}, doi = {10.1109/TMI.2019.2905770}, author = {Hugo Kuijf and Matthijs Biesbroek and Jeroen de Bresser and Rutger Heinen and Simon Andermatt and Mariana Bento and Matt Berseth and Mikhail Belyaev and Jorge Cardoso and Adri{\`a} Casamitjana and Louis Collins and Mahsa Dadar and Achileas Georgiou and Mohsen Ghafoorian and Dakai Jin and April Khademi and Jesse Knight and Hongwei Li and Xavier Llado and Miguel Luna and Qaiser Mahmood and Richard McKinley and Alireza Mehrtash and Sebastien Ourselin and Bo-yong Park and Hyunkin Park and Sang Hyun Park and Simon Pezold and Elodie Puybareau and Leticia Rittner and Carole Sudre and Sergi Valverde and Ver{\'o}nica Vilaplana and Rolan Wiest and Yongchao Xu and Ziyue Xu and Guodong Zeng and Jianguo Zhang and Guoyan Zheng and Christoper Chen and Wiesje van der Flier and Frederik Barkhof and Max Viergever and Geert Jan Biessels} } @mastersthesis {xTubau, title = {Wav2Pix: Enhancement and Evaluation of a Speech-conditioned Image Generator}, year = {2019}, abstract = {

The interest for image synthesis has grown exponentially for the last years. Few years ago, it was invented a very powerful tool for this task: Generative Adversarial Networks (GANs). As its high performance in generating realistic images has been proved, nowadays many researchers are putting the focus on cross-modal learning.

Taking advantage of the huge amount of information we can extract from speech (such as identity, gender or emotional state), in this\  work\  we\  explore\  its\  potential\  to\  generate\  face\  images of a speaker by conditioning a GAN with his/her voice.\  We propose the enhancement and evaluation of a deep neural network that is trained from scratch in an end-to-end fashion, generating a face directly from the raw speech waveform without any additional identity information (e.g reference image or one-hot encoding).

This project focus on the enhancement of a previous model proposed by Francisco Roldan. As a result of a deep analysis on the former project strengths and weaknesses, we present a novel dataset collected for this work, with high-quality videos of ten youtubers with notable expressiveness in both the speech and visual signals. Besides, unlike in the preliminary project, four different techniques are proposed in order to assess the results.

}, author = {Tubau, Miquel}, editor = {Amanda Duarte and Xavier Gir{\'o}-i-Nieto} } @conference {cDuartea, title = {Wav2Pix: Speech-conditioned Face Generation using Generative Adversarial Networks}, booktitle = {ICASSP}, year = {2019}, month = {05/2019}, publisher = {IEEE}, organization = {IEEE}, address = {Brighton, UK}, abstract = {

Speech is a rich biometric signal that contains information\ about the identity, gender and emotional state of the speaker.\ In this work, we explore its potential to generate face images\ of a speaker by conditioning a Generative Adversarial\ Network (GAN) with raw speech input. We propose a deep\ neural network that is trained from scratch in an end-to-end\ fashion, generating a face directly from the raw speech waveform\ without any additional identity information (e.g reference\ image or one-hot encoding). Our model is trained in a\ self-supervised fashion by exploiting the audio and visual signals\ naturally aligned in videos. With the purpose of training\ from video data, we present a novel dataset collected for this\ work, with high-quality videos of ten youtubers with notable\ expressiveness in both the speech and visual signals.

}, doi = {10.1109/ICASSP.2019.8682970}, url = {http://hdl.handle.net/2117/167073}, author = {Amanda Duarte and Rold{\'a}n, Francisco and Tubau, Miquel and Escur, Janna and Pascual-deLaPuente, Santiago and Amaia Salvador and Mohedano, Eva and McGuinness, Kevin and Jordi Torres and Xavier Gir{\'o}-i-Nieto} } @inbook {bTochon18, title = {Advances in utilization of hierarchical representations in remote sensing data analysis}, booktitle = {Reference Module in Earth Systems and Environmental Sciences}, volume = {2}, year = {2018}, pages = {77-107}, publisher = {Elsevier}, organization = {Elsevier}, issn = {978-0-12-409548-9}, author = {G. Tochon and Dalla Mura, M. and Veganzones, M.A. and Valero, S. and Salembier, P. and Chanussot, J.} } @mastersthesis {xRoca, title = {Block-based Speech-to-Speech Translation}, year = {2018}, abstract = {

This bachelor{\textquoteright}s thesis explores different ways of building a block-based Speech Translation system with the aim of generating huge amounts of parallel speech data. The first goal is to research and manage to run suitable tools to implement each one of the three blocks that integrates the Speech Translation system: Speech Recognition, Translation and Speech Synthesis. We experiment with some open-source toolkits and we manage to train a speech recognition system and a neural machine translation system. Then, we test them to evaluate their performance. As an alternative option, we use the cloud computing solutions provided by Google Cloud to implement the three sequential blocks and we successfully build the overall system. Finally, we make a comparative study between an in-house software development versus Cloud computing implementation.

Block-based Speech to Speech Translation from Universitat Polit{\`e}cnica de Catalunya
}, author = {Roca, Sandra}, editor = {Amanda Duarte and Xavier Gir{\'o}-i-Nieto} } @conference {cSuris, title = {Cross-modal Embeddings for Video and Audio Retrieval}, booktitle = {ECCV 2018 Women in Computer Vision Workshop}, year = {2018}, month = {09/2018}, publisher = {Springer}, organization = {Springer}, address = {Munich, Germany}, abstract = {

The increasing amount of online videos brings several opportunities for training self-supervised neural networks. The creation of large scale datasets of videos such as the YouTube-8M allows us to deal with this large amount of data in manageable way. In this work, we find new ways of exploiting this dataset by taking advantage of the multi-modal information it provides. By means of a neural network, we are able to create links between audio and visual documents, by projecting them into a common region of the feature space, obtaining joint audio-visual embeddings. These links are used to retrieve audio samples that fit well to a given silent video, and also to retrieve images that match a given a query audio. The results in terms of Recall@K obtained over a subset of YouTube-8M videos show the potential of this unsupervised approach for cross-modal feature learning. We train embeddings for both scales and assess their quality in a retrieval problem, formulated as using the feature extracted from one modality to retrieve the most similar videos based on the features computed in the other modality.

Cross Modal Embeddings for Video and Audio Retrieval $\#$WiCV18 from Universitat Polit{\`e}cnica de Catalunya
}, isbn = {978-3-030-11018-5}, doi = {10.1007/978-3-030-11018-5_62}, url = {https://doi.org/10.1007/978-3-030-11018-5_62}, author = {Sur{\'\i}s, D{\'\i}dac and Amanda Duarte and Amaia Salvador and Jordi Torres and Xavier Gir{\'o}-i-Nieto} } @article {aDimiccoli18, title = {Introduction to the special issue: Egocentric Vision and Lifelogging}, journal = {Journal of Visual Communication and Image Representation}, year = {2018}, month = {06/2018}, abstract = {

}, doi = {10.1016/j.jvcir.2018.06.010}, url = {https://www.sciencedirect.com/science/article/pii/S1047320318301354}, author = {Dimiccoli, M. and Gurrin, Cathal and Crandall, David and Xavier Gir{\'o}-i-Nieto and Radeva, Petia} } @conference {cDuarte, title = {Towards Speech to Sign Language Translation}, booktitle = {ECCV 2018 Workshop on Shortcomings in Vision and Language}, year = {2018}, month = {08/2018}, abstract = {

Sign Language (SL) is the primary means of communication for a majority of the hearing-impaired community. Current computational approaches in this research area have focused specifically on Sign Language Recognition (SLR) and Sign Language Translation (from SL to text) (SLT). However, the reverse problem of translating from spoken language to sign language has so far been unexplored. The goal of our ongoing project is to provide to people with hearing disabilities the audio tracks from online videos, by automatically generating a video-based speech to sign language translation. In this paper, we will point out the shortcomings that limit the advances of this research area and propose first steps towards this end.

}, author = {Amanda Duarte and Camli, Gorkem and Jordi Torres and Xavier Gir{\'o}-i-Nieto} } @conference {cGurrina, title = {LTA 2017: The Second Workshop on Lifelogging Tools and Applications}, booktitle = {ACM Multimedia}, year = {2017}, month = {10/2017}, publisher = {ACM}, organization = {ACM}, address = {Mountain View, California USA}, abstract = {

The organisation of personal data is receiving increasing research attention due to the challenges we face in gathering, enriching, searching, and visualising such data. \ Given the increasing ease with which personal data being gathered by individuals, the concept of a lifelog digital library of rich multimedia and sensory content for every individual is fast becoming a reality. \ The LTA~2017 workshop aims to bring together academics and practitioners to discuss approaches to lifelog data analytics and applications; and to debate the opportunities and challenges for researchers in this new and challenging area.

}, doi = {10.1145/3123266.3132050}, author = {Gurrin, Cathal and Xavier Gir{\'o}-i-Nieto and Radeva, Petia and Dimiccoli, M. and Dang-Nguyen, Duc-Tien and Joho, H.} } @mastersthesis {xCompri, title = {Multi-label Remote Sensing Image Retrieval based on Deep Features}, year = {2017}, abstract = {

Student: Michele Compri

Advisors: Beg{\"u}m Demir (University of Trento) and Xavier Giro-i-Nieto (UPC)

Recent advances in satellite technology has led to an increased volume of remote sensing (RS) image archives, from which retrieving useful information is challenging. Thus, one important research area in remote sensing (RS) is the content-based retrieval of RS images (CBIR). The performance of the CBIR systems depends on the capability of the RS image features in modeling the content of the images as well as the considered retrieval algorithm that assesses the similarity among the features. Existing CBIR systems in the RS literature assume that each image is categorized by only a single label in terms of a land-cover class that is associated to the most significant content of the image. However, RS images usually have complex content, i.e., there are usually several regions within each image related to multiple land-cover classes. Thus, available CBIR systems are not capable of accurately characterizing and exploiting the high level semantic content of RS images for retrieval problems.

To overcome these problems and to effectively characterize the high-level semantic content of RS images, we investigate effectiveness of different deep learning architectures in the framework of multi-label remote sensing image retrieval problems. This is achieved based on a two-steps strategy. In the first step, aConvolutional Neural Network (CNN) pre-trained for image classification with the ImageNet dataset is used off-the-shelf as a feature extractor. In particular, three popular architectures are explored: 1) VGG16; 2) Inception V3; and 3) ResNet50. VGG16 is a CNN characterized by 16 convolutional layers of stacked 3x3 filters, with intermediate max pooling layers and 3 fully connected layers at the end. Inception V3 is an improved version of the former GoogleNet, which contains more layers but less parameters, by removing fully connected layers and using a global average pooling from the last convolutional layer. ResNet50 is even deeper thanks to the introduction of residual layers, that allow data to flow by skipping the convolutional blocks. In he second step of our research, we modify these three off-the-shelf models by fine-tunning their parameters with a subset of RS images and their multi-label information. Experiments carried out on an archive of aerial images show that fine-tuning CNN architectures with annotated images with multi-labels significantly improve the retrieval accuracy with respect to the standard CBIR methods. We find that fine-tunning using with a multi-class approach achieves better results than than considering each label as an independent class. Due to the space constraints, the detailed information on each step of the proposed method will be given in the full version of the paper.\ 

[source code on GitHub]

Multi-label Remote Sensing Image Retrieval based on Deep Features from Xavier Giro
}, author = {Compri, Michele}, editor = {Demir, Beg{\"u}m and Xavier Gir{\'o}-i-Nieto} } @conference {xLidona, title = {Semantic Summarization of Egocentric Photo Stream Events}, booktitle = {ACM Multimedia 2017 Workshop on Lifelogging Tools and Applications}, year = {2017}, month = {10/2017}, publisher = {ACM}, organization = {ACM}, address = {Mountain View, CA, USA}, abstract = {

With the rapid increase of users of wearable cameras in recent years and of the amount of data they produce, there is a strong need for automatic retrieval and summarization techniques. This work addresses the problem of automatically summarizing egocentric photo streams captured through a wearable camera by taking an image retrieval perspective. After removing non-informative images by a new CNN-based filter, \ images are ranked by relevance to ensure semantic diversity and \ finally re-ranked by a novelty criterion to reduce redundancy. \ To assess the results, a new evaluation metric is proposed which takes into account the non-uniqueness of the solution. Experimental results applied on a database of 7,110 images from 6 different subjects and evaluated by experts gave 95.74\% of experts satisfaction and a Mean Opinion Score of 4.57 out of 5.0.

}, doi = {10.1145/3133202.3133204}, url = {https://dl.acm.org/citation.cfm?id=3133204}, author = {Lidon, Aniol and Bola{\~n}os, Marc and Dimiccoli, M. and Radeva, Petia and Garolera, Maite and Xavier Gir{\'o}-i-Nieto} } @conference {xDuarte, title = {Temporal-aware Cross-modal Embeddings for Video and Audio Retrieval}, booktitle = {NIPS 2017 Women in Machine Learning Workshop (WiML)}, year = {2017}, month = {12/2017}, publisher = {NIPS 2017 Women in Machine Learning Workshop}, organization = {NIPS 2017 Women in Machine Learning Workshop}, address = {Long Beach, CA, USA}, abstract = {

The increasing amount of videos online brings several opportunities for training self-supervised neural networks. In this work, we explore cross-modal embeddings between audio and vision by exploiting their alignment on YouTube videos.

Joint audio-visual embeddings allow creating links between audio and visual documents by projecting them to a common region of the feature space. They can be applied to enriching radio broadcasts with images, finding soundtracks for user-generated videos or simply enriching a topic search with both audio and video documents.

The idea of creating a joint embedding space across modalities has being exploited by other areas [3, 4]. However, joint representation between the video frames and its audio have yet to be fully exploited. A similar approach to the proposed one was [2], where a soundtrack was retrieved to match a music video. However, this work did not target a synchronization between both modalities.

We aim at training a temporal-aware embedding which can align both audio and visual tracks. We use the visual and audio features provided in the YouTube-8M dataset [1]. The dataset includes features at both the clip and frame (temporal window) level. We train embeddings for both scales and assess their quality in a retrieval problem, formulated as using the feature extracted from one modality to retrieve the most similar videos based on the features computed in the other modality.

We aim at not only finding related documents, but synchronize both sequences. The alignment between the two sequences will rely on computing temporal-aware features with recurrent neural networks at different scales. At retrieval time, different scales will be assessed and results evaluated both with ranking metrics and Amazon Mechanical Turk.

References

[1] Sami Abu-El-Haija, Nisarg Kothari, Joonseok Lee, Paul Natsev, George Toderici, Balakrishnan Varadarajan, and Sudheendra Vijayanarasimhan. Youtube-8m: A large-scale video classification benchmark. arXiv preprint arXiv:1609.08675, 2016.

[2] Sungeun Hong, Woobin Im, and Hyun S Yang. Deep learning for content-based, cross-modal retrieval of videos and music. arXiv preprint arXiv:1704.06761, 2017.

[3] Amaia Salvador, Nicholas Hynes, Yusuf Aytar, Javier Marin, Ferda Ofli, Ingmar Weber, and Antonio Torralba. Learning cross-modal embeddings for cooking recipes and food images. In CVPR, 2017.

[4] Liwei Wang, Yin Li, and Svetlana Lazebnik. Learning deep structure-preserving image-text embeddings. In CVPR, 2016.

}, author = {Amanda Duarte and Sur{\'\i}s, D{\'\i}dac and Amaia Salvador and Jordi Torres and Xavier Gir{\'o}-i-Nieto} } @conference {cLe, title = {Towards large scale multimedia indexing: A case study on person discovery in broadcast news}, booktitle = {International Workshop on Content-Based Multimedia Indexing - CBMI 2017}, year = {2017}, month = {06/2017}, address = {Firenze, Italy}, abstract = {

The rapid growth of multimedia databases and the human interest in their peers make indices representing the location and identity of people in audio-visual documents essential for searching archives. Person discovery in the absence of prior identity knowledge requires accurate association of audio-visual cues and detected names. To this end, we present 3 different strategies to approach this problem: clustering-based naming, verification-based naming, and graph-based naming. Each of these strategies utilizes different recent advances in unsupervised face / speech representation, verification, and optimization. To have a better understanding of the approaches, this paper also provides a quantitative and qualitative comparative study of these approaches using the associated corpus of the Person Discovery challenge at MediaEval 2016. From the results of our experiments, we can observe the pros and cons of each approach, thus paving the way for future promising research directions.

}, doi = {https://doi.org/10.1145/3095713.3095732}, author = {Nam Le and Herv{\'e} Bredin and Gabriel Sargent and Miquel India and Paula Lopez-Otero and Claude Barras and Camille Guinaudeau and Guillaume Gravier and Gabriel Barbosa da Fonseca and Izabela Lyon Freire and Zenilton Patroc{\'\i}nio Jr. and Silvio Jamil F. Guimaraes and Gerard Mart{\'\i} and Morros, J.R. and Javier Hernando and Laura Docio-Fernandez and Carmen Garcia-Mateo and Sylvain Meignier and Jean-Marc Odobez} } @conference {cMarsden, title = {Dublin City University and Partners{\textquoteright} Participation in the INS and VTT Tracks at TRECVid 2016}, booktitle = {TRECVID Workshop 2016}, year = {2016}, month = {11/2016}, address = {Gaithersburg, MD, USA}, abstract = {

DCU participated with a consortium of colleagues from NUIG and UPC in two tasks,\ INS and VTT. For the INS task we developed a framework consisting of face detection and\ representation and place detection and representation, with a user annotation of top-ranked\ videos. For the VTT task we ran 1,000 concept detectors from the VGG-16 deep CNN on\ 10 keyframes per video and submitted 4 runs for caption re-ranking, based on BM25, Fusion,\ Word2Vec and a fusion of baseline BM25 and Word2Vec. With the same pre-processing for\ caption generation we used an open source image-to-caption CNN-RNN toolkit NeuralTalk2\ to generate a caption for each keyframe and combine them.

}, url = {http://doras.dcu.ie/21484/}, author = {Marsden, Mark and Mohedano, Eva and McGuinness, Kevin and Calafell, Andrea and Xavier Gir{\'o}-i-Nieto and O{\textquoteright}Connor, N. and Zhou, Jiang and Azevedo, Lucas and Daubert, Tobias and Davis, Brian and H{\"u}rlimann, Manuela and Afli, Haithem and Du, Jinhua and Ganguly, Debasis and Li, Wei and Way, Andy and Smeaton, Alan F.} } @conference {cdeOliveira-Barraa, title = {Large Scale Content-Based Video Retrieval with LIvRE}, booktitle = {14th International Workshop on Content-based Multimedia Indexing (CBMI)}, year = {2016}, month = {06/2016}, publisher = {IEEE}, organization = {IEEE}, address = {Bucharest, Romania}, abstract = {

The fast growth of video data requires robust, efficient, and scalable systems to allow for indexing and retrieval. These systems must be accessible from lightweight, portable and usable interfaces to help users in management and search of video content. This demo paper presents LIvRE, an extension of an existing open source tool for image retrieval to support video indexing. LIvRE consists of three main system components (pre-processing, indexing and retrieval), as well as a scalable and responsive HTML5 user interface accessible from a web browser. LIvRE supports image-based queries, which are efficiently matched with the extracted frames of the indexed videos.

}, author = {Gabriel de Oliveira-Barra and Lux, Mathias and Xavier Gir{\'o}-i-Nieto} } @conference {cdeOliveira-Barra, title = {LEMoRe: A Lifelog Engine for Moments Retrieval at the NTCIR-Lifelog LSAT Task}, booktitle = {The 12th NTCIR Conference, Evaluation of Information Access Technologies}, year = {2016}, month = {06/2016}, publisher = {National Institute of Informatics (NII)}, organization = {National Institute of Informatics (NII)}, address = {Tokyo, Japan}, abstract = {

Semantic image retrieval from large amounts of egocentric visual data requires to leverage powerful techniques for filling in the semantic gap. This paper introduces LEMoRe, a Lifelog Engine for Moments Retrieval, developed in the context of the Lifelog Semantic Access Task (LSAT) of the the NTCIR-12 challenge and discusses its performance variation on different trials. \ LEMoRe integrates classical image descriptors with high-level semantic concepts extracted by Convolutional Neural Networks (CNN), powered by a graphic user interface that uses natural language processing. \ Although this is just a first attempt towards interactive image retrieval from large egocentric datasets and there is a large room for improvement of the system components and the user interface, the structure of the system itself and the way the single components cooperate are very promising.

LEMoRe - A Lifelog Engine for Moments Retrieval at NTCIR-12 from University of Barcelona
}, url = {http://research.nii.ac.jp/ntcir/workshop/OnlineProceedings12/NTCIR/toc_ntcir.html$\#$Lifelog}, author = {Gabriel de Oliveira-Barra and Xavier Gir{\'o}-i-Nieto and Cartas-Ayala, Alejandro and Radeva, Petia} } @conference {cGurrin, title = {LTA 2016 - The First Workshop on Lifelogging Tools and Applications}, booktitle = {ACM Multimedia}, year = {2016}, month = {10/2016}, publisher = {ACM}, organization = {ACM}, address = {Amsterdam, The Netherlands}, abstract = {

The organisation of personal data is receiving increasing research attention due to the challenges that are faced in gathering, enriching, searching and visualising this data. Given the increasing quantities of personal data being gathered by individuals, the concept of a lifelog digital library of rich multimedia and sensory content for every individual is fast becoming a reality. The LTA2016 lifelogging workshop at ACM MM 2016 aims to bring together academics and practitioners to discuss approaches to lifelog data analytics and the applications of same, and to debate the opportunities and challenges for researchers in this new and challenging area.

[Workshop web page]

[Workshop proceedings]

[UPCommons]

}, keywords = {lifelogging, Personal digital archives, Personal information management}, doi = {http://dx.doi.org/10.1145/2964284.2980534}, url = {http://lta2016.computing.dcu.ie/}, author = {Gurrin, Cathal and Xavier Gir{\'o}-i-Nieto and Radeva, Petia and Dimiccoli, M. and Johansen, H. and Joho, H. and Singh, Vivek K} } @mastersthesis {xBarra, title = {LIvRE: A Video Extension to the LIRE Content-Based Image Retrieval System}, year = {2015}, abstract = {

Advisors:\ Mathias Lux\ (Klagenfurt University) and\ Xavier Gir{\'o}-i-Nieto\ (UPC)

Degree: Electronic Engineering (5 years) at\ Telecom BCN-ETSETB\ (UPC)

Grade: A (9.0/10.0)

This project explores the expansion of Lucene Image Retrieval Engine (LIRE), an open-source Content-Based Image Retrieval (CBIR) system, for video retrieval on large scale video datasets. The fast growth of the need to store huge amounts of video in servers requires efficient, scalable search and indexing engines capable to assist users in their management and retrieval. In our tool, queries are formulated by visual examples allowing users to find the videos and the moment of time when the query image is matched with. The video dataset used on this scenario comprise over 1,000 hours of different news broadcast channels. This thesis presents an extension and adaptation of Lire and its plugin for Solr, an open-source enterprise search platform from the Apache Lucene project, for video retrieval based on visual features, as well as a web-interface for users from different devices.

LIvRE: A Video Extension to the LIRE Content-Based Image Retrieval System from Xavier Giro
}, url = {http://upcommons.upc.edu/handle/2117/79052}, author = {Gabriel de Oliveira-Barra}, editor = {Xavier Gir{\'o}-i-Nieto and Lux, Mathias} } @inbook {bDigne15, title = {Neighborhood Filters and the Recovery of 3D Information}, booktitle = {Handbook of Mathematical Methods in Imaging}, year = {2015}, pages = {1645-1673}, publisher = {Springer Verlag}, organization = {Springer Verlag}, edition = {2}, isbn = {978-1-4939-0789-2}, author = {J. Digne and Dimiccoli, M. and N. Sabater and Salembier, P.} } @conference {cNguyen, title = {NII-HITACHI-UIT at TRECVID 2015 Instance Search}, booktitle = {TRECVID 2015 Workshop}, year = {2015}, month = {11/2015}, publisher = {NIST}, organization = {NIST}, address = {Gaithersburg, MD, USA}, abstract = {

In this paper, we propose two methods to improve last year instance search framework. Both of them are based on post processing scheme that try to rerank top K shots returned from BOW model. The rst system is to propose a query-adaptive weighting technique between DPM object detectors score and BOW{\textquoteright}s score. In order to nd a good weight, we use a neural network which learns characteristics of the query including number of features, number of shared words and area of the query topic. The second system combines two state-of-the-art object detectors: DPM and Fast RCNN to estimate object location and similarity score, respectively. The nal score is computed using these components together with BOW based similarity score returned from the baseline system. The experimental results show that our system improved pretty much even with a smaller number of top K input ranked list. Compared to other teams, we got the second place with the same run.

}, url = {http://www-nlpir.nist.gov/projects/tvpubs/tv.pubs.15.org.html}, author = {Nguyen, Vinh-Tiep and Duy-Dinh-Le and Amaia Salvador and Caizhi-Zhu and Nguyen, Dinh-Luan and Tran, Minh-Triet and Ngo Duc, Thanh and Anh Duong, Duc and Satoh, Shin{\textquoteright}ichi and Xavier Gir{\'o}-i-Nieto} } @conference {cVentura13, title = {Automatic Keyframe Selection based on Mutual Reinforcement Algorithm}, booktitle = {CBMI (Content-Based Multimedia Indexing)}, year = {2013}, month = {09/2013}, address = {Veszprem}, abstract = {

This paper addresses the problem of video summarization through an automatic selection of a single representative keyframe. The proposed solution is based on the mutual reinforcement paradigm, where a keyframe is selected thanks to its highest and most frequent similarity to the rest of considered frames. Two variations of the algorithm are explored: a first one where only frames within the same video are used (intraclip mode) and a second one where the decision also depends on the previously selected keyframes of related videos (interclip mode). These two algorithms were evaluated by a set of
professional documentalists from a broadcaster{\textquoteright}s archive, and results concluded that the proposed techniques outperform the semi-manual solution adopted so far in the company.

http://cbmi2013.mik.uni-pannon.hu/

Automatic Keyframe Selection based on Mutual Reinforcement Algorithm from Xavi Gir{\'o}
}, keywords = {mutual reinforcement algorithm, video summarization}, isbn = {978-1-4799-0955-1}, doi = {10.1109/CBMI.2013.6576548}, url = {http://dx.doi.org/10.1109/CBMI.2013.6576548}, author = {Ventura, C. and Xavier Gir{\'o}-i-Nieto and Ver{\'o}nica Vilaplana and Daniel Giribet and Eusebio Carasusan} } @article {aAlonso-Gonzalez13a, title = {Bilateral Distance Based Filtering for Polarimetric SAR Data}, journal = {Remote Sensing}, volume = {5}, year = {2013}, month = {10/2013}, pages = {5620-5641}, abstract = {

This paper introduces a non-linear Polarimetric SAR data filtering approach able to preserve the edges and small details of the data. It is based on exploiting the data locality in both, the spatial and the polarimetric domains, in order to avoid mixing heterogeneous samples of the data. A weighted average is performed over a given window favoring pixel values that are close on both domains. The filtering technique is based on a modified bilateral filtering, which is defined in terms of spatial and polarimetric distances. These distances encapsulate all the knowledge in both domains for an adaptation to the data structure. Finally, the proposed technique is employed to process a real RADARSAT-2 dataset.

}, issn = {2072-4292}, doi = {10.3390/rs5115620}, author = {Alonso-Gonz{\'a}lez, A. and L{\'o}pez-Mart{\'\i}nez, C. and Salembier, P. and X. Deng} } @conference {cJimenez12, title = {Registration of Multi-Modal Neuroimaging Datasets by Considering the Non-Overlapping Field of View into the NMI Calculation}, booktitle = {IEEE International Symposium on Biomedical Imaging, ISBI 2012}, year = {2012}, address = {Barcelona, Spain}, author = {Jim{\'e}nez, X and Figueiras, F and Marqu{\'e}s, F. and Salembier, P. and Herance, R and Rojas, S and Mill{\'a}n, O and Pareto, D and Domingo Gispert, J} } @conference {cBenediktsson11, title = {Hierarchical analysis of remote sensing data: morphological attribute profiles and binary partition trees}, booktitle = {International Symposium on Mathematical Morphology 2011}, year = {2011}, pages = {306{\textendash}319}, address = {Intra, Lake Maggiore, Italy}, isbn = {3642215688}, doi = {10.1007/978-3-642-21569-8_27}, url = {http://hdl.handle.net/2117/14624}, author = {Benediktsson, J. and Bruzzone, L. and Chanussot, J. and Dalla Mura, M. and Salembier, P. and Valero, S.} } @inbook {bDigne11, title = {Neighborhood Filters and the Recovery of 3D Information}, booktitle = {Handbook of Mathematical Methods in Imaging}, year = {2011}, pages = {1203-1229}, publisher = {Springer Verlag}, organization = {Springer Verlag}, isbn = {978-0-387-92919-4}, author = {J. Digne and Dimiccoli, M. and Salembier, P. and N. Sabater} } @book {eDutoit09, title = {Applied signal processing: A Matlab-based proof of concept}, year = {2009}, publisher = {Springer}, organization = {Springer}, isbn = {978-0-387-74534-3}, url = {http://cataleg.upc.edu/record=b1353617~S1*cat}, author = {Dutoit, T. and Marqu{\'e}s, F.} } @conference {cDimiccoli09, title = {Exploiting T-junctions for depth segregation in single images}, booktitle = {IEEE International Conference on Acoustics, Speech, and Signal Processing, ICASSP 2009}, year = {2009}, address = {Taipei, Taiwan}, author = {Dimiccoli, M. and Salembier, P.} } @article {Caselles200916, title = {Flux-gradient and source-term balancing for certain high resolution shock-capturing schemes}, journal = {Computers \& Fluids}, volume = {38}, year = {2009}, month = {0/2009}, pages = {16 - 36}, abstract = {

We present an extension of Marquina{\textquoteright}s flux formula, as introduced in Fedkiw et al. [Fedkiw RP, Merriman B, Donat R, Osher S. The penultimate scheme for systems of conservation laws: finite difference \{ENO\} with Marquina{\textquoteright}s flux splitting. In: Hafez M, editor. Progress in numerical solutions of partial differential equations, Arcachon, France; July 1998], for the shallow water system. We show that the use of two different Jacobians at cell interfaces prevents the scheme from satisfying the exact C-property [Berm{\'u}dez A, V{\'a}zquez ME. Upwind methods for hyperbolic conservation laws with source terms. Comput Fluids 1994;23(8):1049{\textendash}71] while the approximate C-property is satisfied for higher order versions of the scheme. The use of a single Jacobian in Marquina{\textquoteright}s flux splitting formula leads to a numerical scheme satisfying the exact C-property, hence we propose a combined technique that uses Marquina{\textquoteright}s two sided decomposition when the two adjacent states are not close and a single decomposition otherwise. Finally, we propose a special treatment at wet/dry fronts and situations of dry bed generation.

}, issn = {0045-7930}, doi = {10.1016/j.compfluid.2007.07.023}, author = {Caselles, Vicent and Donat, Rosa and Haro, G.} } @conference {cDimiccoli09a, title = {Hierarchical region-based representation for segmentation and filtering with depth in single images}, booktitle = {IEEE International Conference on Image Processing, ICIP 2009}, year = {2009}, address = {Cairo, Egypt}, author = {Dimiccoli, M. and Salembier, P.} } @inbook {bMarques09, title = {How are digital images compressed in the web?}, booktitle = {Applied signal processing}, year = {2009}, pages = {265{\textendash}310}, isbn = {978-0-387-74534-3}, url = {http://cataleg.upc.edu/record=b1353617~S1*cat}, author = {Marqu{\'e}s, F. and Menezes, M. and Ruiz-Hidalgo, J.}, editor = {Dutoit, T. and Marqu{\'e}s, F.} } @inbook {bMarques09a, title = {How are digital TV programs compressed to allow broadcasting?}, booktitle = {Applied signal processing}, year = {2009}, pages = {311{\textendash}359}, isbn = {978-0-387-74534-3}, url = {http://cataleg.upc.edu/record=b1353617~S1*cat}, author = {Marqu{\'e}s, F. and Menezes, M. and Ruiz-Hidalgo, J.}, editor = {Dutoit, T. and Marqu{\'e}s, F.} } @inbook {bBach09, title = {How can physicians quantify brain degeneration?}, booktitle = {Applied signal processing}, year = {2009}, pages = {411{\textendash}449}, isbn = {978-0-387-74534-3}, url = {http://cataleg.upc.edu/record=b1353617~S1*cat}, author = {Bach, M. and Thiran, J. and Marqu{\'e}s, F.}, editor = {Dutoit, T. and Marqu{\'e}s, F.} } @inbook {bDescamps09, title = {How does digital cinema compress images?}, booktitle = {Applied signal processing}, year = {2009}, pages = {361{\textendash}410}, isbn = {978-0-387-74534-3}, url = {http://cataleg.upc.edu/record=b1353617~S1*cat}, author = {Descamps, A. and De Vleeschouwer, C. and Jacques, L. and Marqu{\'e}s, F.}, editor = {Dutoit, T. and Marqu{\'e}s, F.} } @inbook {bDimakis09, title = {The Memory Jog Service}, booktitle = {Computers in the Human Interaction Loop}, year = {2009}, pages = {207{\textendash}234}, publisher = {Springer}, organization = {Springer}, address = {London}, abstract = {

The CHIL Memory Jog service focuses on facilitating the collaboration of participants in meetings, lectures, presentations, and other human interactive events, occurring in indoor CHIL spaces. It exploits the whole set of the perceptual components that have been developed by the CHIL Consortium partners (e.g., person tracking, face identification, audio source localization, etc) along with a wide range of actuating devices such as projectors, displays, targeted audio devices, speakers, etc.

}, isbn = {978-1-84882-053-1}, doi = {10.1007/978-1-84882-054-8}, url = {http://www.springerlink.com/content/tg220751v5m1h356/}, author = {Dimakis, N. and Soldatos, J. and Polymenakos, L. and Sturm, J. and Neumann, J. and Casas, J.} } @phdthesis {dDimiccoli09, title = {Monocular Depth Estimation for Image Segmentation and Filtering}, year = {2009}, school = {Universitat Polit{\`e}cnica de Catalunya (UPC)}, type = {phd}, author = {Dimiccoli, M.}, editor = {Salembier, P.} } @conference {cDimiccoli08, title = {Monocular Depth by Nonlinear Diffusion}, booktitle = {IEEE Sixth Indian Conference on Computer Vision, Graphics \& Image Processing, ICVGIP 2008}, year = {2008}, pages = {140{\textendash}148}, address = {Bhubaneswar, India}, author = {Dimiccoli, M. and Morel, J. and Salembier, P.} } @book {eAguilar07, title = {Diccionari de Telecomunicacions}, year = {2007}, isbn = {978-84-412-1459-0}, author = {Aguilar, M. and Alcober, J. and Altes, J. and Aragones, X. and Artigas, D. and Bardes, D. and Barlabe, A. and Bragos, R. and Calderer, J. and Cardama, A. and Casademont, J. and Casals, L. and Comer{\'o}n, A. and Cotrina, J. and Cruz, L. and Federico Dios and Duxans, H. and Esparza, O. and Esquerra, I. and Garcia, D. and Garcias, P. and Gomez, C. and Gorricho, J. and Guinjoan, F. and Hesselbach, X. and Liria, A. and Lopez, J. and Madrenas, J. and Madue{\~n}o, M. and Mestre, F. and Monte, E. and Morros, J.R. and Mu{\~n}oz, J. and Pallar{\'e}s, E. and Pons, J. and Recolons, J. and Rincon, D. and Riu, P. and Pradell, L. and Pascual-Iserte, A. and Prat, L. and Rey, F. and Villares, J.} } @conference {cPetras07, title = {Flexible test-bed for unusual behavior detection}, booktitle = {6th ACM International Conference on Image and Video Retrieval}, year = {2007}, pages = {105{\textendash}108}, isbn = {0 646 34170 7}, author = {Petras, I. and Beleznai, C. and Dedeoglu, Y. and M. Pard{\`a}s and Kov{\'a}cs, L. and Szl{\'a}vik, Z. and Havasi, L. and Szir{\'a}nyi, T. and T{\"o}reyin, B. and G{\"u}d{\"u}kbay, U. and Cetin, E. and Cristian Canton-Ferrer} } @conference {cDimiccoli07a, title = {Geometrical Filtering Scheme with Connected Operators and Image Inpainting}, booktitle = {SPIE Visual Communcations and Image Processing 2007}, year = {2007}, pages = {1{\textendash}14}, address = {San Jose, CA, USA}, author = {Dimiccoli, M. and Salembier, P.} } @conference {cDorea07, title = {Hierarchical partition-based representations for image sequences using trajectory merging criteria}, booktitle = {2007 International Conference on Acoustics, Speech and Signal Processing}, year = {2007}, author = {Dorea, C. and M. Pard{\`a}s and Marqu{\'e}s, F.} } @conference {cDorea07a, title = {A hierarchical trajectory-based representation for video}, booktitle = {Fith International Workshop on Content-Based Multimedia Indexing}, year = {2007}, pages = {275{\textendash}282}, isbn = {1-4244-1011-8}, author = {Dorea, C. and M. Pard{\`a}s and Marqu{\'e}s, F.} } @conference {cDimiccoli07, title = {Perceptual filtering with connected operators and image inpainting}, booktitle = {Proc. of International Symposium on Mathematical Morphology, ISMM 2007}, year = {2007}, pages = {227{\textendash}238}, address = {Rio de Janeiro, Brazil}, isbn = {978-85-17-00032-4}, author = {Dimiccoli, M. and Salembier, P.} } @article {aDescampe07, title = {Prefetching and caching strategies for remote and interactive browsing of JPEG2000 images}, journal = {IEEE transactions on image processing}, volume = {16}, number = {5}, year = {2007}, pages = {1339{\textendash}1354}, issn = {1057-7149}, author = {Descampe, A. and Vleeschouwer, D. and Iregui, M. and Macq, B. and Marqu{\'e}s, F.} } @conference {cDorea06, title = {Generation of long-term color and motion coherent partitions}, booktitle = {International Conference on Image Processing}, year = {2006}, pages = {581{\textendash}584}, author = {Dorea, C. and M. Pard{\`a}s and Marqu{\'e}s, F.} } @conference {cDescampe06, title = {Pre-fetching strategies for remote and interactive browsing of JPEG2000 images}, booktitle = {International Conference on Image Processing}, year = {2006}, pages = {3203{\textendash}3206}, author = {Descampe, A. and De Vleeschouwer, C. and Iregui, M. and Macq, B. and Marqu{\'e}s, F.} } @inbook {bPardas04, title = {The InterFace Software Platform for Interactive Virtual Characters}, booktitle = {Mpeg-4 facial animation: the standard, implementation and applications}, year = {2004}, pages = {169{\textendash}183}, isbn = {0-470-84465-5}, author = {M. Pard{\`a}s and Pandzic, I. and Cannella, M. and Davoine, F. and Forchheimer, R. and Lavagetto, F. and Marriott, A. and Malassiotis, S.} } @conference {cDorea04, title = {A region-based algorithm for image segmentation and parametric motion estimation}, booktitle = {Image Analysis for Multimedia Interactive Services}, year = {2004}, isbn = {972-98115-7-1}, author = {Dorea, C. and M. Pard{\`a}s and Marqu{\'e}s, F.} } @conference {cTorres94, title = {Segmentation based coding of textures using stochastic vector quantization}, booktitle = {IEEE International Conference on Acoustics, Speech and Signal Processing}, year = {1994}, pages = {553{\textendash}556}, abstract = {

In second generation image compression techniques the image to be compressed is first segmented. The pixels are divided into mutually exclusive spatial regions based on some criteria. After segmentation, the image consists of regions separated by contours. Then, the information is coded describing the shapes and interiors of the regions. The interiors of the regions are usually encoded using polynomials. The objective of this paper is to encode the interior of the regions by stochastic vector quantization techniques. If the segmentation process has been well defined and the obtained regions are homogeneous, then it is possible to design a specific codebook suited to the statistics of each region. The approach is to design the codebook according to some previously defined model for the regions of the image found in the segmentation process. If the approach is combined with efficient contour coding techniques, good visual results for high compression rates are obtained

}, isbn = {84-600-7766-7}, doi = {10.1109/ICASSP.1994.389380}, author = {Torres, L. and Casas, J. and Diego, D.} }