@article {aPachon-Garcia, title = {SurvLIMEpy: A Python package implementing SurvLIME}, journal = {Expert Systems With Applications}, volume = {237, Part C}, year = {2024}, month = {03/2024}, abstract = {

In this paper we present SurvLIMEpy, an open-source Python package that implements the SurvLIME algorithm. This method allows to compute local feature importance for machine learning algorithms designed for modelling Survival Analysis data. Our\ implementation takes advantage of the parallelisation paradigm as all computations are performed in a matrix-wise fashion which speeds up execution time. Additionally, SurvLIMEpy assists the user with visualization tools to better understand the result of the algorithm. The package supports a wide variety of survival models, from the Cox Proportional Hazards Model to deep learning models such as DeepHit or DeepSurv. Two types of experiments are presented in this paper. First, by means of simulated data, we study the ability of the algorithm to capture the importance of the features. Second, we use three open source survival datasets together with a set of survival algorithms in order to demonstrate how SurvLIMEpy behaves when applied to different models.

}, keywords = {eXplainable Artificial Intelligence, Interpretable Machine Learning, machine learning, python, Survival Analysis}, author = {Cristian Pach{\'o}n-Garc{\'\i}a and Carlos Hernandez and Pedro Delicado and Ver{\'o}nica Vilaplana} } @conference {cTarresa, title = {Sign Language Translation from Instructional Videos}, booktitle = {CVPR 2023 Women in Computer Vision Workshop}, year = {2023}, month = {04/2023}, publisher = {Computer Vision Foundation / IEEE}, organization = {Computer Vision Foundation / IEEE}, address = {Vancouver, Canada}, abstract = {

The advances in automatic sign language translation (SLT) to spoken languages have been mostly benchmarked with datasets of limited size and restricted domains. Our work advances the state of the art by providing the first baseline results on How2Sign, a large and broad dataset. We train a Transformer over I3D video features, using the reduced BLEU as a reference metric for validation, instead of the widely used BLEU score. We report a result of 8.03 on the BLEU score, and publish the first open-source implementation of its kind to promote further advances.

}, author = {Laia Tarr{\'e}s and Gerard I. G{\'a}llego and Amanda Duarte and Jordi Torres and Xavier Gir{\'o}-i-Nieto} } @article {aFerrer-Ferrer, title = {Simultaneous Fruit Detection and Size Estimation Using Multitask Deep Neural Networks }, journal = {Biosystems Engineering}, volume = {233}, year = {2023}, month = {09/2023}, pages = {63-75}, abstract = {

The measurement of fruit size is of great interest to estimate the yield and predict the harvest resources in advance. This work proposes a novel technique for in-field apple detection and measurement based on Deep Neural Networks. The proposed\ \ framework was trained with RGB-D data and consists of an end-to-end multitask Deep Neural Network architecture 13 specifically designed to perform the following tasks: 1) detection and segmentation of each fruit from its surroundings; 2) estimation of the diameter of each detected fruit. The methodology was tested with a total of 15335 annotated apples at different growth stages, with diameters varying from 27 mm to 95 mm. Fruit detection results reported an F1-score for apple detection of 0.88 and a mean absolute error of diameter estimation of 5.64 mm. These are state-of-the-art results with the additional advantages of: a) using an end-to-end multitask trainable network; b) an efficient and fast inference speed; and c) being based on RGB-D data which can be acquired with affordable depth cameras. On the contrary, the main disadvantage is the need of annotating a large amount of data with fruit masks and diameter ground truth to train the model. Finally, a fruit visibility analysis showed an improvement in the prediction when limiting the measurement to apples above 65\% of visibility (mean absolute error of 5.09 mm). This suggests that future works should develop a method for automatically identifying the most visible apples and discard the prediction of highly occluded fruits.

}, keywords = {deep learning, Fruit measurement, Fruit visibility, Precision agriculture, Yield estimation}, doi = {https://doi.org/10.1016/j.biosystemseng.2023.07.010}, author = {Ferrer-Ferrer, M. and Ruiz-Hidalgo, J. and Gregorio, Eduard and Ver{\'o}nica Vilaplana and Morros, J.R. and Gen{\'e}-Mola, Jordi} } @conference {cCaselles, title = {SIRA: Relightable Avatars from a Single Image}, booktitle = {Winter Conference on Applications of Computer Vision (WACV)}, year = {2023}, abstract = {

Recovering the geometry of a human head from a single image, while factorizing the materials and illumination is a severely ill-posed problem that requires prior information to be solved. Methods based on 3D Morphable Models (3DMM), and their combination with differentiable renderers, have shown promising results. However, the expressiveness of 3DMMs is limited, and they typically yield over-smoothed and identity-agnostic 3D shapes limited to the face region. Highly accurate full head reconstructions have recently been obtained with neural fields that parameterize the geometry using multilayer perceptrons. The versatility of these representations has also proved effective for disentangling geometry, materials and lighting. However, these methods require several tens of input images. In this paper, we introduce SIRA, a method which, from a single image, reconstructs human head avatars with high fidelity geometry and factorized lights and surface materials. Our key ingredients are two data-driven statistical models based on neural fields that resolve the ambiguities of single-view 3D surface reconstruction and appearance factorization. Experiments show that SIRA obtains state of the art results in 3D head reconstruction while at the same time it successfully disentangles the global illumination, and the diffuse and specular albedos. Furthermore, our reconstructions are amenable to physically-based appearance editing and head model relighting.

}, author = {Caselles, Pol and Ramon, Eduard and Garcia, Jaime and Xavier Gir{\'o}-i-Nieto and Moreno, Francesc and Triginer, Gil} } @article {aPardas23, title = {Stromal tissue segmentation in Ki67 histology images based on cytokeratin-19 stain translation}, journal = {JOURNAL OF MEDICAL IMAGING}, volume = {10}, year = {2023}, month = {06/2023}, abstract = {

Purpose

The diagnosis and prognosis of breast cancer relies on histopathology image analysis. In this context, proliferation markers, especially Ki67, are increasingly important. The diagnosis using these markers is based on the quantification of proliferation, which implies the counting of Ki67 positive and negative tumoral cells in epithelial regions, thus excluding stromal cells. However, stromal cells are often very difficult to distinguish from negative tumoral cells in Ki67 images and often lead to errors when automatic analysis is used.

Approach

We study the use of automatic semantic segmentation based on convolutional neural networks (CNNs) to separate stromal and epithelial areas on Ki67 stained images. CNNs need to be accurately trained with extensive databases with associated ground truth. As such databases are not publicly available, we propose a method to produce them with minimal manual labelling effort. Inspired by the procedure used by pathologists, we have produced the database relying on knowledge transfer from cytokeratin-19 images to Ki67 using an image-to-image (I2I) translation network.

Results

The automatically produced stroma masks are manually corrected and used to train a CNN that predicts very accurate stroma masks for unseen Ki67 images. An F-score value of 0.87 is achieved. Examples of effect on the KI67 score show the importance of the stroma segmentation.

Conclusions

An I2I translation method has proved very useful for building ground-truth labeling in a task where manual labeling is unfeasible. With reduced correction effort, a dataset can be built to train neural networks for the difficult problem of separating epithelial regions from stroma in stained images where separation is very hard without additional information.

}, isbn = {2329-4302}, doi = {https://doi.org/10.1117/1.JMI.10.3.037502}, author = {M. Pard{\`a}s and D. Anglada and M. Espina and F. Marques and Salembier, P.} } @conference {cHurtado23, title = {Study of Manifold Geometry using Multiscale Non-Negative Kernel Graphs}, booktitle = {IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)}, year = {2023}, month = {06/2023}, address = {Rhodes Island, Greece}, isbn = {978-1-7281-6327-7}, doi = {https://doi.org/10.1109/ICASSP49357.2023.10095956}, author = {Hurtado, C. and Shekkizhar, S. and Ruiz-Hidalgo, J. and Ortega, A.} } @article {aBernal22, title = {SALAI-Net: species-agnostic local ancestry inference network}, journal = {Bioinformatics}, volume = {38}, year = {2022}, month = {09/2022}, chapter = {ii27}, abstract = {

Motivation
Local ancestry inference (LAI) is the high resolution prediction of ancestry labels along a DNA sequence. LAI is important in the study of human history and migrations, and it is beginning to play a role in precision medicine applications including ancestry-adjusted genome-wide association studies (GWASs) and polygenic risk scores (PRSs). Existing LAI models do not generalize well between species, chromosomes or even ancestry groups, requiring re-training for each different setting. Furthermore, such methods can lack interpretability, which is an important element in each of these applications.

Results
We present SALAI-Net, a portable statistical LAI method that can be applied on any set of species and ancestries (species-agnostic), requiring only haplotype data and no other biological parameters. Inspired by identity by descent methods, SALAI-Net estimates population labels for each segment of DNA by performing a reference matching approach, which leads to an interpretable and fast technique. We benchmark our models on whole-genome data of humans and we test these models{\textquoteright} ability to generalize to dog breeds when trained on human data. SALAI-Net outperforms previous methods in terms of balanced accuracy, while generalizing between different settings, species and datasets. Moreover, it is up to two orders of magnitude faster and uses considerably less RAM memory than competing methods.

}, doi = {https://doi.org/10.1093/bioinformatics/btac464}, url = {https://academic.oup.com/bioinformatics/article/38/Supplement_2/ii27/6701999?login=false$\#$373158629}, author = {Bernal, Oriol and Mas-Montserrat, Daniel and Xavier Gir{\'o}-i-Nieto and Ioannidis, Alexander G.} } @article {aSalgueirob, title = {SEG-ESRGAN: A multi-task network for super-resolution and semantic segmentation of remote sensing images}, journal = {Remote Sensing}, volume = {14}, year = {2022}, month = {2022}, chapter = {5862}, abstract = {

The production of highly accurate land cover maps is one of the primary challenges in remote sensing, which depends on the spatial resolution of the input images. Sometimes, high-resolution imagery is not available or is too expensive to cover large areas or to perform multitemporal analysis. In this context, we propose a multi-task network to take advantage of the freely available Sentinel-2 imagery to produce a super-resolution image, with a scaling factor of 5, and the corresponding high-resolution land cover map. Our proposal, named SEG-ESRGAN, consists of two branches: the super-resolution branch, that produces Sentinel-2 multispectral images at 2 m resolution, and an encoder{\textendash}decoder architecture for the semantic segmentation branch, that generates the enhanced land cover map. From the super-resolution branch, several skip connections are retrieved and concatenated with features from the different stages of the encoder part of the segmentation branch, promoting the flow of meaningful information to boost the accuracy in the segmentation task. Our model is trained with a multi-loss approach using a novel dataset to train and test the super-resolution stage, which is developed from Sentinel-2 and WorldView-2 image pairs. In addition, we generated a dataset with ground-truth labels for the segmentation task. To assess the super-resolution improvement, the PSNR, SSIM, ERGAS, and SAM metrics were considered, while to measure the classification performance, we used the IoU, confusion matrix and the F1-score. Experimental results demonstrate that the SEG-ESRGAN model outperforms different full segmentation and dual network models (U-Net, DeepLabV3+, HRNet and Dual_DeepLab), allowing the generation of high-resolution land cover maps in challenging scenarios using Sentinel-2 10 m bands.

}, issn = {2072-4292 }, doi = {https://doi.org/10.3390/rs142}, url = {https://doi.org/10.3390/rs14225862 }, author = {Luis Salgueiro and Javier Marcello and Ver{\'o}nica Vilaplana} } @conference {cPinaa, title = {Self-supervised graph representations of WSIs}, booktitle = {Geometric Deep Learning in Medical Image Analysis}, year = {2022}, month = {2022}, abstract = {

In this manuscript we propose a framework for the analysis of whole slide images (WSI) on the cell entity space with self-supervised deep learning on graphs and explore its representation quality at different levels of application. It consists of a two step process in which the cell level analysis is performed locally, by clusters of nearby cells that can be seen as small regions of the image, in order to learn representations that capture the cell environment and distribution. In a second stage, a WSI graph is generated with these regions as nodes and the representations learned as initial node embeddings. The graph is leveraged for a downstream task, region of interest (ROI) detection addressed as a graph clustering. The representations outperform the evaluation baselines at both levels of application, which has been carried out predicting whether a cell, or region, is tumor or not based on its learned representations with a logistic regressor.

}, url = {https://proceedings.mlr.press/v194/pina22a/pina22a.pdf}, author = {Oscar Pina and Ver{\'o}nica Vilaplana} } @conference {cHernandeza, title = {Sentinel lymph node status prediction using self-attention networks and contrastive learning from routine histology images of primary tumours}, booktitle = {Medical Imaging with Deep Learning MIDL 2022}, year = {2022}, month = {07/2022}, abstract = {

Deep learning-based computational pathology approaches are becoming increasingly prominent in histopathology image analysis. However, these images typically come with drawbacks that hamper automatic analysis, which include: labeled sample scarcity or the extremely large size of the images (ranging from\ \ to\ \ pixels). Nonetheless, they have proven to be a powerful tool for diagnosis and risk prevention. One such prevention is reducing the number of patients who undergo surgeries that do not benefit them. This study develops a pipeline for predicting sentinel lymph node (SLN) metastasis non-invasively from digitised Whole Slide Images (WSI) of primary melanoma tumours. Furthermore, we combine the use of a weakly supervised architecture with self-supervised contrastive pre-training. We experimentally demonstrate that 1) the use of self-attention improves sentinel lymph node status prediction and 2) self-supervised contrastive learning improves the quality of the learned representations compared to a standard ImageNet pre-training, which boosts the model{\textquoteright}s performance.

}, author = {Carlos Hernandez and Marc Combalia and Josep Malvehy and Ver{\'o}nica Vilaplana} } @conference {cHernandez22, title = {Sentinel lymph node status prediction with self-attention neural networks using histologies of primary melanoma tumours}, booktitle = {European Association of Dermato Oncology (EADO 2022)}, year = {2022}, month = {04/2022}, author = {Carlos Hernandez and Ver{\'o}nica Vilaplana and Marc Combalia and Sergio Garc{\'\i}a and Sebastian Podlipnik and Julio Burgos and Susana Puig and Josep Malvehy} } @article {xCabot22, title = {Sign Language Translation based on Transformers for the How2Sign Dataset}, year = {2022}, abstract = {

Introduction to Research, BSc Data Science and Engineering, Autumn 2021:

The end goal of Sign Language Translation is to either produce spoken sentences from sign videos or generate sign videos from their corresponding written transcriptions. In this situation, this task has been address in multiple approaches in recent years. Moreover, it has been proved that taking advantage of the sign gloss representations improves substantially the model{\textquoteright}s performance in this task. Therefore, in this work we replicate the state-of-the-art Transformer-based approach on the task and evaluate it on the multimodal American Sign Language How2Sign dataset. Furthermore, we provide baseline recognition and translation results that represent an starting point to further research on the topic. In addition, we provide a new sentence-based alignment for the How2Sign videos, as their current alignment was with speech, which we have used to tackle the Sign Language Translation task properly.\ 

}, author = {Patricia Cabot}, editor = {Laia Tarr{\'e}s and Xavier Gir{\'o}-i-Nieto} } @conference {cDuartec, title = {Sign Language Video Retrieval with Free-Form Textual Queries}, booktitle = {CVPR 2022 - CVF/IEEE Conference on Computer Vision and Pattern Recognition}, year = {2022}, abstract = {

Systems that can efficiently search collections of sign language videos have been highlighted as a useful application of sign language technology. However, the problem of searching videos beyond individual keywords has received limited attention in the literature. To address this gap, in this work we introduce the task of sign language retrieval with free-form textual queries: given a written query (e.g., a sentence) and a large collection of sign language videos, the objective is to find the signing video in the collection that best matches the written query. We propose to tackle this task by learning cross-modal embeddings on the recently introduced large-scale How2Sign dataset of American Sign Language (ASL). We identify that a key bottleneck in the performance of the system is the quality of the sign video embedding which suffers from a scarcity of labeled training data. We, therefore, propose SPOT-ALIGN, a framework for interleaving iterative rounds of sign spotting and feature alignment to expand the scope and scale of available training data. We validate the effectiveness of SPOT-ALIGN for learning a robust sign video embedding through improvements in both sign recognition and the proposed video retrieval task.

}, author = {Amanda Duarte and Samuel Albanie and Xavier Gir{\'o}-i-Nieto and G{\"u}l Varol} } @mastersthesis {xCabot22a, title = {Sign-Language Translation with Pseudo-Glosses}, year = {2022}, abstract = {

Sign Language Translation is an open problem whose goal is to generate written sentences from sign videos. In recent years, many research works that have been developed in this field mainly addressed the Sign Language Recognition task, which consists in understanding the input signs and transcribing them into sequences of annotations. Moreover, current studies show that taking advantage of the latter task helps to learn meaningful representations and can be seen as an intermediate step towards the end goal of translation.


In this work, we present a method to generate automatic pseudo-glosses from written sentences, which can work as a replacement for real glosses. This addresses the issue of their collection, as they need to be manually annotated and it is extremely costly.

Furthermore, we introduce a new implementation built on Fairseq of the Transformer-model approach introduced by Camgoz et al., which is jointly trained to solve the recognition and translation tasks. Besides, we provide new baseline results on both implementations: first, on the Phoenix dataset, we present results that outperform the ones provided by Camgoz et al. in their work, and, second, on the How2Sign dataset, we present the first results on the translation task. These results can work as a baseline for future research in the field.

Patricia Cabot with her advisors Laia Tarr{\'e}s, Gerard I. G{\'a}llego and Xavier Gir{\'o}-i-Nieto.
}, author = {Patricia Cabot and Laia Tarr{\'e}s and Xavier Gir{\'o}-i-Nieto} } @conference {cMosella-Montoro22, title = {SkinningNet: Two-Stream Graph Convolutional Neural Network for Skinning Prediction of Synthetic Characters}, booktitle = {IEEE / CVF Computer Vision and Pattern Recognition Conference (CVPR)}, year = {2022}, month = {06/2022}, address = {New Orleans, USA}, abstract = {

This work presents SkinningNet, an end-to-end Two-Stream Graph Neural Network architecture that computes skinning weights from an input mesh and its associated skeleton, without making any assumptions on shape class and structure of the provided mesh. Whereas previous methods pre-compute handcrafted features that relate the mesh and the skeleton or assume a fixed topology of the skeleton, the proposed method extracts this information in an end-to-end learnable fashion by jointly learning the best relationship between mesh vertices and skeleton joints. The proposed method exploits the benefits of the novel Multi-Aggregator Graph Convolution that combines the results of different aggregators during the summarizing step of the Message-Passing scheme, helping the operation to generalize for unseen topologies. Experimental results demonstrate the effectiveness of the contributions of our novel architecture, with SkinningNet outperforming current state-of-the-art alternatives.

https://imatge-upc.github.io/skinningnet/

}, url = {https://imatge-upc.github.io/skinningnet/}, author = {Mosella-Montoro, Albert and Ruiz-Hidalgo, J.} } @conference {cPina, title = {Structural Networks for Brain Age Prediction}, booktitle = {Medical Imaging with Deep Learning (MIDL 2022)}, year = {2022}, month = {08/2022}, abstract = {

Biological networks have gained considerable attention within the Deep Learning community because of the promising framework of Graph Neural Networks (GNN), neural models that operate in complex networks. In the context of neuroimaging, GNNs have successfully been employed for functional MRI processing but their application to ROI-level structural MRI (sMRI) remains mostly unexplored. In this work we analyze the implementation of these geometric models with sMRI by building graphs of ROIs (ROI graphs) using tools from Graph Signal Processing literature and evaluate their performance in a downstream supervised task, age prediction. We first make a qualitative and quantitative comparison of the resulting networks obtained with common graph topology learning strategies. In a second stage, we train GNN-based models for brain age prediction. Since the order of every ROI graph is exactly the same and each vertex is an entity by itself (a ROI), we evaluate whether including ROI information during message-passing or global pooling operations is beneficial and compare the performance of GNNs against a Fully-Connected Neural Network baseline. The results show that ROI-level information is needed during the global pooling operation in order to achieve competitive results. However, no relevant improvement has been detected when it is incorporated during the message passing. These models achieve a MAE of 4.27 in hold-out test data, which is a performance very similar to the baseline, suggesting that the inductive bias included with the obtained graph connectivity is relevant and useful to reduce the dimensionality of the problem

}, author = {Oscar Pina and Irene Cumplido-Mayoral and Raffaele Cacciaglia and Jos{\'e} Mar{\'\i}a Gonz{\'a}lez-de-Ech{\'a}varri and Juan D. Gispert and Ver{\'o}nica Vilaplana} } @phdthesis {dSalgueiro22, title = {Super-resolution and semantic segmentation of remote sensing images using deep learning techniques}, volume = {PhD}, year = {2022}, month = {10/2022}, abstract = {

Remote sensing for Earth observation is a growing scientific field essential for many human activities. Among the different applications in the Remote Sensing domain, the production of thematic maps, such as Land Cover and Land Use, are among the most relevant, as this information plays a critical role in management, planning and monitoring activities at different levels. In this context, the Sentinel-2 satellites are of great importance since they provide open data on land and coastal areas at different spatial resolutions (10, 20, and 60 m), democratizing usability, and creating a high potential for the generation of valuable information, useful in many scenarios, such as agriculture, forestry, land cover and urban planning, among others.

In this thesis, we aim to exploit the usability of Sentinel-2 data by applying deep learning techniques, which are revolutionizing the world of computer vision and, recently, remote sensing. First, we propose super-resolution models to improve the spatial details of the different Sentinel-2 bands, and second, we propose the conjunction of semantic segmentation with super-resolution to generate improved land cover maps that benefit from the enhanced spatial details of the bands.

We first address super-resolution by proposing two different models, one for the 10 m/pixel bands to reach 2 m/pixel and another for the 20 and 60 m/pixel bands to achieve 10 m/pixel. Then, we propose two different multitasking models to derive land cover maps. The first one extending a semantic segmentation model to produce an additional super-resolution image and the second, improving our first super-resolution approach, to provide a semantic segmentation map, as well. We combine features of the different tasks within a single model to improve performance and to generate a high-resolution image with the corresponding highquality land cover map. All models developed were evaluated, quantitatively and qualitatively, using different datasets, showing excellent performance in diverse complex scenarios.

}, author = {Luis Salgueiro}, editor = {Ver{\'o}nica Vilaplana and Javier Marcello} } @conference {cManas, title = {Seasonal Contrast: Unsupervised Pre-Training from Uncurated Remote Sensing Data}, booktitle = {International Conference in Computer Vision (ICCV)}, year = {2021}, publisher = {IEEE/CVF}, organization = {IEEE/CVF}, address = {Virtual}, abstract = {

Remote sensing and automatic earth monitoring are key to solve global-scale challenges such as disaster prevention, land use monitoring, or tackling climate change. Although there exist vast amounts of remote sensing data, most of it remains unlabeled and thus inaccessible for supervised learning algorithms. Transfer learning approaches can reduce the data requirements of deep learning algorithms. However, most of these methods are pre-trained on ImageNet and their generalization to remote sensing imagery is not guaranteed due to the domain gap. In this work, we propose Seasonal Contrast (SeCo), an effective pipeline to leverage unlabeled data for in-domain pre-training of re-mote sensing representations. The SeCo pipeline is com-posed of two parts. First, a principled procedure to gather large-scale, unlabeled and uncurated remote sensing datasets containing images from multiple Earth locations at different timestamps. Second, a self-supervised algorithm that takes advantage of time and position invariance to learn transferable representations for re-mote sensing applications. We empirically show that models trained with SeCo achieve better performance than their ImageNet pre-trained counterparts and state-of-the-art self-supervised learning methods on multiple downstream tasks. The datasets and models in SeCo will be made public to facilitate transfer learning and enable rapid progress in re-mote sensing applications.

}, url = {https://arxiv.org/abs/2103.16607}, author = {Ma{\~n}as,Oscar and Lacoste, Alexandre and Xavier Gir{\'o}-i-Nieto and Vazquez, David and Rodr{\'\i}guez, Pau} } @article {xGiro-i-Nieto21, title = {Sign Language Translation and Production Multimedia and Multimodal Challenges for All}, year = {2021}, abstract = {

Machine translation and computer vision have greatly benefited of the advances in deep learning. The large and diverse amount of textual and visual data have been used to train neural networks whether in a supervised or self-supervised manner. Nevertheless, the convergence of the two field in sign language translation and production is still poses multiple open challenges, like the low video resources, limitations in hand pose estimation, or 3D spatial grounding from poses. This talk will present these challenges and the How2V️Sign dataset recorded at CMU in collaboration with UPC, BSC, Gallaudet University and Facebook.



}, author = {Xavier Gir{\'o}-i-Nieto} } @article {aSalgueiroa, title = {Single-image super-resolution of Sentinel-2 low resolution bands with residual dense convolutional neural networks}, journal = {Remote Sensing}, volume = {13}, year = {2021}, month = {2021}, pages = {5007}, abstract = {

Sentinel-2 satellites have become one of the main resources for\  Earth observation images because they are free of charge, have a great spatial coverage and high temporal revisit. Sentinel-2 senses the same location providing different spatial resolutions as well as generating a multi-spectral image with 13 bands of 10, 20, and 60 m/pixel. In this work, we propose a single-image super-resolution model based on convolutional neural networks that enhances the low-resolution bands (20 m and 60 m) to reach the maximal resolution sensed (10 m) at the same time, whereas other approaches provide two independent models for each group of LR bands. Our proposed model, named Sen2-RDSR, is made up of Residual in Residual blocks that produce two final outputs at maximal resolution, one for 20 m/pixel bands and the other for 60 m/pixel bands. The training is done in two stages, first focusing on 20 m bands and then on the 60 m bands. Experimental results using 6 quality metrics (RMSE, SRE, SAM, PSNR, SSIM, ERGAS) show that our model has superior performance compared to other state-of-the-art approaches, and it is very effective and suitable as a preliminary step for land and coastal applications, as studies involving pixel-based classification for Land-Use-Land-Cover or the generation of vegetation indices.

}, author = {Luis Salgueiro and Javier Marcello and Ver{\'o}nica Vilaplana} } @conference {cPuig-Sitjes21, title = {Spatio-temporal Detection and Tracking of Thermal Events on the Plasma Facing Components of Wendelstein 7-X}, booktitle = {4th IAEA Technical Meeting on Fusion Data Processing, Validation and Analysis}, year = {2021}, month = {11/2021}, publisher = {iaea.org}, organization = {iaea.org}, address = {Shanghai (online)}, abstract = {

In steady-state fusion devices like Wendelstein 7-X (W7-X), the active control of heat loads is mandatory to attain long-plasma operation. An intelligent feedback control system that mitigates the risk of overheating is required to avoid a premature plasma termination by the safety system. To keep the plasma within the safe operational limits of the plasma facing components, the feedback control system must be informed of the ongoing thermal events and their evolution in time. Then it can take effectives countermeasures to prevent the thermal events from reaching a critical point. These countermeasures may include reducing the heating power, changing the strike-line position or inducing detachment. With reaction times of the order of a hundred milliseconds, a fully automated real-time image analysis algorithm is required.
In this work, we present a spatio-temporal algorithm to detect, classify and track the thermal events observed by the thermography diagnostic on the plasma facing components of W7-X. The system detects and distinguishes between strike-lines and isolated hot spots as well as leading edges. The segmentation of the strike-line is specially challenging at W7-X. As a 3-dimensional helically-shaped stellarator equipped with 10 island divertors, the strike-lines have a complex heat load distribution with a high-dynamic range. The use of morphological tools and, in particular, the use of the Max-tree transform allow us to segment the thermal events in a hierarchical way preserving the inclusion relationship between different events, like hot spots and leading edges embedded in the strike-line structure. The thermal events are segmented for each frame and tracked over time in order to forecast their temporal evolution and to evaluate their risk. To this end, a spatio-temporal graph is built and spatio-temporal connected components are used to track the thermal events across the sequence frames. The spatio-temporal components in the graph are used to label the events in the sequence preserving temporal coherence and minimizing discontinuities, solving splits and merges. Spatio-temporal descriptors are then generated for each event to assess their risk.
The algorithm was tested offline on the infrared data acquired during the last operation phase OP1.2 and the results are presented here. Further work will follow to accelerate the code with GPUs to reach real-time processing and be ready to protect the water-cooled plasma facing components in the forthcoming operation phase OP2.

}, url = {https://conferences.iaea.org/event/251/contributions/20704/}, author = {Puig-Sitjes, A. and Casas, J. and Salembier, P. and Vizcarro, D. and Clemente, R. and Jakubowski, M. and Gao, Y. and W7-X Team} } @mastersthesis {xOriola, title = {Species-agnostic Local Ancestry Inference on Genomic Data with Convolutions}, year = {2021}, abstract = {

Local Ancestry Inference (LAI) is the high resolution prediction of ancestry (African, European, ...) across a DNA sequence. LAI is becoming increasingly important in DNA sequence analysis for the study of human ancestry and migrations. It is also necessary for polygenic risk scores research (prediction of traits and disease risk). Most current LAI models are built for specific species, set of ancestries and chromosomes, hence a new model needs to be trained from scratch for every slightly different setting. This creates a big barrier for research and industry to shift across different LAI scenarios. In this thesis we present SALAI-Net, the first statistical method for LAI with reference panel that can be used on any set of species and ancestries (species-agnostic). Loter is the state of the art in species-agnostic models with reference panel, and is based on a dynamic programming algorithm. However, it is slow and does not perform very well in small reference panel settings. Our model is based on a novel hand-engineered template matching block followed by a convolutional smoothing filter optimized to minimize cross-entropy loss on a training dataset. The right choice of DNA sequence encoding, similarity features and architecture is what makes our model able to generalize well to unseen ancestries, species, and different chromosomes. We benchmark our models on whole genome data of humans and we test the ability to generalize to dog species when trained on human data. Our models outperform the state-of-the-art method by a big margin in terms of accuracy, testing in different settings and datasets. Moreover, it is up to two orders of magnitude faster. Our model also shows close to no generalization gap when switching between species.

}, author = {Oriol, Benet}, editor = {Mas-Montserrat, Daniel and Ioannidis, Alexander G. and Xavier Gir{\'o}-i-Nieto} } @phdthesis {dGirbau21, title = {Sports broadcasting and multiple object tracking with deep learning methods}, volume = {PhD}, year = {2021}, month = {03/2021}, type = {Industrial}, abstract = {

Since less than a decade ago, deep learning techniques started to dominate many different\ fields, revolutionizing the possibilities of artificial intelligence. Seeing their potential, industrial\ sectors started to invest in applying such technologies as key components of the company\ strategy. This thesis has been developed in an industrial context, in AutomaticTV. The main\ focus along this period has been the transfer of knowledge and know-how between academia\ and industry, development of tools to exploit this knowledge, the exploration of new techniques\ for future challenges, and, from an academic research perspective, contributions to the\ multiple object tracking problem.

The first part of the thesis is devoted to the introduction of deep learning technologies to\ AutomaticTV, a company dedicated to automatic sports analysis and broadcasting, and the\ development of tools and tasks that surround the application.

The second part of this thesis introduces the contributions to the multiple object tracking\ challenge. We present TrajE, a trajectory estimator based on mixture density networks and\ beam search, used to boost the performance of existing multiple object trackers, and introduce\ an occlusion reconstruction step using the estimated trajectory information. By adding TrajE\ to an existing multiple object tracker, we boost its performance by 6.3, 1.8 points in MOTA and\ IDF1 scores respectively, becoming the new state of the art in the MOTChallenge dataset.

}, author = {Girbau, A.}, editor = {Rius, Ignasi and Xavier Gir{\'o}-i-Nieto and Marqu{\'e}s, F.} } @conference {cKazakos, title = {SynthRef: Generation of Synthetic Referring Expressions for Object Segmentation}, booktitle = {NAACL Visually Grounded Interaction and Language (ViGIL) Workshop}, year = {2021}, month = {06/2021}, address = {Virtual}, abstract = {

Recent advances in deep learning have brought significant progress in visual grounding tasks such as language-guided video object segmentation. However, collecting large datasets for these tasks is expensive in terms of annotation time, which represents a bottleneck. To this end, in our work we propose a novel method, namely SynthRef, for generating synthetic referring expressions for an image (or video frame), and we also present and disseminate the first large-scale dataset with synthetic referring expressions for video object segmentation. Our experiments demonstrate that by training with our synthetic dataset one can improve the ability of a model to generalize across different datasets, without any additional annotation cost. Moreover, our formulation allows its application to any object detection or segmentation dataset.

}, author = {Kazakos, Ioannis and Bellver-Bueno, M{\'\i}riam and Ventura, C. and Silberer, Carina and Xavier Gir{\'o}-i-Nieto} } @mastersthesis {xManas, title = {Self-Supervised Visual Representation Learning for Remote Sensing}, year = {2020}, abstract = {

With the creation of large-scale annotated datasets such as the ImageNet, fully-supervised machine learning methods have become the standard for solving computer vision tasks. These methods require large amounts of labeled data, which are usually obtained with crowdsourcing tools or social media tags. However, these approaches do not scale for specialized domains, such as medical or satellite imaging, where annotations must be provided by experts at a prohibitive cost. Recently, self-supervised learning has emerged as an alternative for obtaining transferable visual representations from unlabeled data. Models based on these representations match the performance of fully-supervised models while only requiring a small fraction of the annotations. In this work, we aim to explore the application of self-supervised learning methods in the remote sensing domain. We propose a contrastive approach for learning visual representations by exploiting the multi-spectral information of satellite images. These representations serve as a good starting point for a variety of downstream tasks that involve remote sensing imagery, accelerating convergence with fewer labeled examples.

Best thesis award 2020 (draw with 4 more other works)\ 

}, author = {Ma{\~n}as,Oscar}, editor = {Rodr{\'\i}guez, Pau and Xavier Gir{\'o}-i-Nieto} } @conference {cPuig-Sitjes20, title = {Strategy for the real-time detection of thermal events on the plasma facing components of Wendelstein 7-X}, booktitle = {31st Symposium on Fusion Technology (SOFT2020)}, year = {2020}, month = {09/2020}, address = {Dubrovnik, Croatia}, abstract = {

Wendelstein 7-X (W7-X), the most advanced fusion experiment in the stellarator line, aims at demonstrating the feasibility of the stellarator concept as a future fusion power plant. It is planned to restart operation by the end of 2021 with a high heat flux divertor and water-cooled plasma facing components (PFCs) to demonstrate steady-state operation. With plasma energy limits starting at 1 GJ and gradually increasing to 18 GJ over several experimental campaigns, the PFCs have to be protected from overheating. For that, a fully autonomous system is required in order to prevent damage to the plasma facing components due to thermal events.
During the last experimental campaign, when W7-X was equipped with inertially cooled test divertor units, extensive experience was gained with the preliminary design of the thermal event detection system. By then, the system was not yet real-time capable and it was not fully automated, requiring manual supervision between discharges. This experience, however, allowed to prove the validity of some design concepts and to define the new strategy towards the protection of the machine in steady-state operation, when the system will be connected to the Interlock System and the feedback control.
In this work, the design of the real-time thermal event detection system for W7-X for steady-state operation is presented. The system is based on the thermography and video diagnostics to monitor the divertor units, the baffles, and the wall heat-shields and panels. It will be implemented on a real-time system and integrated in CoDaC{\textquoteright}s safety infrastructure. The system relies on computer vision and machine learning techniques to perform a spatio-temporal analysis to detect and classify the thermal events and to perform a risk evaluation. The results and the main conclusions drawn from the analysis of the data from the past campaign are reported.

}, url = {http://hdl.handle.net/21.11116/0000-0007-7FF5-7}, author = {Puig-Sitjes, A. and Jakubowski, M. and Fellinger, J. and Drewelow, P. and Gao, Y. and Niemann, H. and Sunn-Pedersen, T. and K{\"o}nig, R. and Naujoks, D. and Winter, A. and Laqua, H. and Dumke, S. and Moncada, V. and Belafdil, C. and Mitteau, R. and Aumeunier, M.-H. and Pisano, F. and Aymerich, E. and Cannas, B. and Kocsis, G. and Szepesi, T. and Cseh, G. and Szabolics, T. and Casas, J. and Morros, J.R. and Salembier, P. and Clemente, R. and Cobos, M. and I. Caminal and Palacios Corral, A. and Moreno Punzano, A. and Quiceno Lopera, S.} } @article {aSalgueiro, title = {Super-Resolution of Sentinel-2 Imagery Using Generative Adversarial Networks}, journal = {Remote Sensing}, volume = {12}, year = {2020}, month = {01/2020}, abstract = {

Sentinel-2 satellites provide multi-spectral optical remote sensing images with four bands at 10 m of spatial resolution. These images, due to the open data distribution policy, are becoming an important resource for several applications. However, for small scale studies, the spatial detail of these images might not be sufficient. On the other hand, WorldView commercial satellites offer multi-spectral images with a very high spatial resolution, typically less than 2 m, but their use can be impractical for large areas or multi-temporal analysis due to their high cost. To exploit the free availability of Sentinel imagery, it is worth considering deep learning techniques for single-image super-resolution tasks, allowing the spatial enhancement of low-resolution (LR) images by recovering high-frequency details to produce high-resolution (HR) super-resolved images. In this work, we implement and train a model based on the Enhanced Super-Resolution Generative Adversarial Network (ESRGAN) with pairs of WorldView-Sentinel images to generate a super-resolved multispectral Sentinel-2 output with a scaling factor of 5. Our model, named RS-ESRGAN, removes the upsampling layers of the network to make it feasible to train with co-registered remote sensing images. Results obtained outperform state-of-the-art models using standard metrics like PSNR, SSIM, ERGAS, SAM and CC. Moreover, qualitative visual analysis shows spatial improvements as well as the preservation of the spectral information, allowing the super-resolved Sentinel-2 imagery to be used in studies requiring very high spatial resolution.

}, author = {Luis Salgueiro and Javier Marcello and Ver{\'o}nica Vilaplana} } @conference {cGarcia-Gomez19, title = {Self-registered lidar and polarimetric images in real-time: application to detection of small objects at sea}, booktitle = {7th Workshop on Active Imaging}, year = {2019}, month = {11/2019}, publisher = {French-German Research Institute of Saint-Louis (ISL)}, organization = {French-German Research Institute of Saint-Louis (ISL)}, address = {Saint-Louis, France}, author = {Pablo Garc{\'\i}a-G{\'o}mez and Jordi Riu and Casas, J. and S. Royo} } @article {aCasamitjanaa, title = {Shared latent structures between imaging features and biomarkers in early stages of Alzheimer{\textquoteright}s disease: a predictive study}, journal = {IEEE Journal of Biomedical and Health Informatics}, year = {2019}, month = {08/2019}, abstract = {

Magnetic resonance imaging (MRI) provides high resolution brain morphological information and is used as a biomarker in neurodegenerative diseases. Population studies of brain morphology often seek to identify pathological structural changes related to different diagnostic categories (e.g: controls, mild cognitive impairment or dementia) which normally describe highly heterogeneous groups with a single categorical variable. Instead, multiple biomarkers are used as a proxy for pathology and are more powerful in capturing structural variability. Hence, using the joint modeling of brain morphology and biomarkers, we aim at describing structural changes related to any brain condition by means of few underlying processes. In this regard, we use a multivariate approach based on Projection to Latent Structures in its regression variant (PLSR) to study structural changes related to aging and AD pathology. MRI volumetric and cortical thickness measurements are used for brain morphology and cerebrospinal fluid (CSF) biomarkers (t-tau, p-tau and amyloid-beta) are used as a proxy for AD pathology. By relating both sets of measurements, PLSR finds a low-dimensional latent space describing AD pathological effects on brain structure. The proposed framework allows to separately model aging effects on brain morphology as a confounder variable orthogonal to the pathological effect. The predictive power of the associated latent spaces (i.e. the capacity of predicting biomarker values) is assessed in a cross-validation framework.\ 

}, keywords = {CSF biomarkers, Latent model, MRI, PLS, preclinical AD}, doi = {10.1109/JBHI.2019.2932565}, author = {Adri{\`a} Casamitjana and Paula Petrone and J.L. Molinuevo and Juan D. Gispert and Ver{\'o}nica Vilaplana} } @article {aSalembier18, title = {Ship Detection in SAR Images Based on Maxtree Representation and Graph Signal Processing}, journal = {IEEE Transactions on Geoscience and Remote Sensing}, volume = {57}, year = {2019}, month = {05/2019}, pages = {2709 - 2724}, author = {Salembier, P. and Liesegang, S. and L{\'o}pez-Mart{\'\i}nez, C.} } @conference {cLinardosa, title = {Simple vs complex temporal recurrences for video saliency prediction}, booktitle = {British Machine Vision Conference (BMVC)}, year = {2019}, month = {09/2019}, publisher = {British Machine Vision Association}, organization = {British Machine Vision Association}, address = {Cardiff, Wales / UK.}, abstract = {

This paper investigates modifying an existing neural network architecture for static saliency prediction using two types of recurrences that integrate information from the temporal domain. The first modification is the addition of a ConvLSTM within the architecture, while the second is a computationally simple exponential moving average of an internal convolutional state. We use weights pre-trained on the SALICON dataset and fine-tune our model on DHF1K. Our results show that both modifications achieve state-of-the-art results and produce similar saliency maps.

}, url = {https://bmvc2019.org/wp-content/uploads/papers/0952-paper.pdf}, author = {Linardos, Panagiotis and Mohedano, Eva and Nieto, Juan Jos{\'e} and O{\textquoteright}Connor, N. and Xavier Gir{\'o}-i-Nieto and McGuinness, Kevin} } @article {aKuijf, title = {Standardized Assessment of Automatic Segmentation of White Matter Hyperintensities; Results of the WMH Segmentation Challenge}, journal = {IEEE Transactions on Medical Imaging}, year = {2019}, month = {03/2019}, abstract = {

Quantification of white matter hyperintensities (WMH) of presumed vascular origin is of key\ importance in many neurological research studies. Advanced\ measurements are obtained from manual segmentations on brain\ MR images, which is a laborious procedure. Automatic WMH\ segmentation methods exist, but a standardized comparison of\ such methods is lacking. We organized a scientific challenge, in\ which developers could evaluate their method on a standardized\ multi-center/-scanner image dataset, giving an objective comparison:\ the WMH Segmentation Challenge (http://wmh.isi.uu.nl/).\ Sixty T1+FLAIR images from three MR scanners were released\ with manual WMH segmentations. A secret test set of 110\ images from five MR scanners was used for evaluation. Methods\ had to be containerized and submitted to the challenge organizers.Five evaluation metrics were used to rank the methods:\ (1) Dice Similarity Coefficient, (2) modified Hausdorff distance\ (95th percentile), (3) absolute percentage volume difference, (4)\ sensitivity for detecting individual lesions, and (5) F1-score for\ individual lesions. Additionally, methods were ranked on their\ inter-scanner robustness.\ Twenty participants submitted their method for evaluation.\ This paper provides a detailed analysis of the results. In brief,there is a cluster of four methods that rank significantly better\ than the other methods. There is one clear winner, which also\ has the best inter-scanner robustness.\ The challenge remains open for future submissions and provides\ a public platform for method evaluation.

}, keywords = {brain, Evaluation and performance, Magnetic resonance imaging (MRI), segmentation}, issn = {0278-0062}, doi = {10.1109/TMI.2019.2905770}, author = {Hugo Kuijf and Matthijs Biesbroek and Jeroen de Bresser and Rutger Heinen and Simon Andermatt and Mariana Bento and Matt Berseth and Mikhail Belyaev and Jorge Cardoso and Adri{\`a} Casamitjana and Louis Collins and Mahsa Dadar and Achileas Georgiou and Mohsen Ghafoorian and Dakai Jin and April Khademi and Jesse Knight and Hongwei Li and Xavier Llado and Miguel Luna and Qaiser Mahmood and Richard McKinley and Alireza Mehrtash and Sebastien Ourselin and Bo-yong Park and Hyunkin Park and Sang Hyun Park and Simon Pezold and Elodie Puybareau and Leticia Rittner and Carole Sudre and Sergi Valverde and Ver{\'o}nica Vilaplana and Rolan Wiest and Yongchao Xu and Ziyue Xu and Guodong Zeng and Jianguo Zhang and Guoyan Zheng and Christoper Chen and Wiesje van der Flier and Frederik Barkhof and Max Viergever and Geert Jan Biessels} } @phdthesis {dCasamitjana19, title = {Study of early stages of Alzheimer{\textquoteright}s disease using magnetic resonance imaging}, volume = {PhD}, year = {2019}, school = {Universitat Polit{\`e}cnica de Catalunya}, address = {Barcelona}, abstract = {

Alzheimer{\textquoteright}s disease (AD) is a neurodegenerative disorder that constitutes the most common dementia pathology. It represents a global epidemic that expands exponentially as the life expectancy increases with no yet useful treatment. Currently, it represents a huge social and economic burden for our societies and it is expected to tension public health infraestructures and finances in the near future.

AD is characterized by amyloid plaque deposition and neurofibrillary tangles measured by ex-vivo examination of the brain. Recent developments in fluid biomarkers and brain imaging allow in-vivo quantification of pathophysiological processes of amyloid deposition or tau tangles formation in the brain, providing the community with highly sensitive and specific in-vivo biomarkers for Alzheimer{\textquoteright}s disease diagnosis. Abnormal levels of these biomarkers are thought as the initiating event to a cascade of subsequent events that continue with synapse loss, cell death, memory impairment, functional dysfunction and cognitive decline. All these events constitute the Alzheimer{\textquoteright}s continuum which can be broadly split into two main parts: an initial long and silent preclinical stage characterized by abnormal AD biomarkers and cognition within the normal range that could last from 15 to 30 years and a posterior clinical stage where subjects develop dementia symptoms.

The etiology of AD is still poorly understood even though several risk factors are identified. Large observational studies can help the study of AD and its related biomarkers and risk factors. In this thesis we provide methodological tools for the analysis of Alzheimer{\textquoteright}s disease using magnetic resonance imaging (MRI). We focus on the study of subjects within the preclinial stage of AD by using statistical learning and pattern recognition frameworks to perform inferential statistics and develop predictive models.

The main outcomes of this thesis are three-fold: firstly, we develop an open-source toolbox for nonlinear neuroimage analysis in population studies. While nonlinear association between medical images and several factors is already known, standard neuroimaging softwares only provide linear statistical frameworks that limit the analyses. Secondly, we study the relationship between brain structure using MRI and the underlying Alzheimer{\textquoteright}s pathology along the disease continuum and at different stages. The close relationship between MRI and clinical symptoms has been widely studied but describing AD using biomarkers instead of clinical phenotypes allows us to study preclinical stages of AD. Finally, we present a framework to predict cognitively unimpaired and amyloid positive subjects using MR imaging and machine learning. We report the results in a cross-sectional study and in a longitudinal study that compares the volumetric rate-of-change between subjects with different amyloid status. We further test the proposed methodology as a part of the triaging process in clinical trials showing great potential benefits.

}, url = {http://hdl.handle.net/10803/668251}, author = {Adri{\`a} Casamitjana}, editor = {Ver{\'o}nica Vilaplana} } @mastersthesis {xCasals19, title = {Synthesis of acne images for data augmentation with generative adversarial networks}, year = {2019}, abstract = {

Generative Adversarial Networks (GANs) are deep learning architectures known for their usefulness on synthesizing new images. Conditioned image generation or the synthesis of super-resolution images are some of their main uses, but they are also helpful when tackling particular image classification and segmentation problems. The latter application is the motivation for the work presented in this document.

This work studies the synthesis of acne images for data augmentation, a procedure validated using said synthetic images to tackle an image classification problem.

The main challenge will be to work around the instability in the training of GANs. Therefore, different known solutions will be implemented in order to overcome this problem.

}, author = {Roger Casals}, editor = {Ver{\'o}nica Vilaplana} } @conference {cMohedanob, title = {Saliency Weighted Convolutional Features for Instance Search}, booktitle = {Content-Based Multimedia Indexing - CBMI}, year = {2018}, month = {09/2018}, publisher = {IEEE}, organization = {IEEE}, address = {La Rochelle, France}, abstract = {

This work explores attention models to weight the contribution of local convolutional representations for the instance search task. We present a retrieval framework based on bags of local convolutional features (BLCF) that benefits from saliency weighting to build an efficient image representation. The use of human visual attention models (saliency) allows significant improvements in retrieval performance without the need to conduct region analysis or spatial verification, and without requiring any feature fine tuning. We investigate the impact of different saliency models, finding that higher performance on saliency benchmarks does not necessarily equate to improved performance when used in instance search tasks. The proposed approach outperforms the state-of-the-art on the challenging INSTRE benchmark by a large margin, and provides similar performance on the Oxford and Paris benchmarks compared to more complex methods that use off-the-shelf representations.

Saliency Weighted Convolutional Features for Instance Search from Universitat Polit{\`e}cnica de Catalunya
}, author = {Mohedano, Eva and McGuinness, Kevin and Xavier Gir{\'o}-i-Nieto and O{\textquoteright}Connor, N.} } @article {aAssens, title = {Scanpath and Saliency Prediction on 360 Degree Images}, journal = {Elsevier Signal Processing: Image Communication}, year = {2018}, abstract = {

We introduce deep neural networks for scanpath and saliency prediction trained on 360-degree images. The scanpath prediction model called SaltiNet is based on a temporal-aware novel representation of saliency information named the saliency volume. The first part of the network consists of a model trained to generate saliency volumes, whose parameters are fit by back-propagation using a binary cross entropy (BCE) loss over downsampled versions of the saliency volumes. Sampling strategies over these volumes are used to generate scanpaths over the 360-degree images. Our experiments show the advantages of using saliency volumes, and how they can be used for related tasks. We also show how a similar architecture achieves state-of-the-art performance for the related task of saliency map prediction. Our source code and trained models available here.

}, url = {https://www.sciencedirect.com/science/article/pii/S0923596518306209}, author = {Assens, Marc and McGuinness, Kevin and O{\textquoteright}Connor, N. and Xavier Gir{\'o}-i-Nieto} } @phdthesis {dLin18, title = {Semantic and Generic Object Segmentation for Scene Analysis Using RGB-D Data}, year = {2018}, month = {07/2018}, school = {Universitat Polit{\`e}cnica de Catalunya (UPC)}, type = {PhD}, address = {download link}, abstract = {

In this thesis, we study RGB-D based segmentation problems from different per- spectives in terms of the input data. Apart from the basic photometric and geometric information contained in the RGB-D data, also semantic and temporal information are usually considered in an RGB-D based segmentation system.

The first part of this thesis focuses on an RGB-D based semantic segmentation problem, where the predefined semantics and annotated training data are available. First, we review how RGB-D data has been exploited in the state-of-the-art to help training classifiers in semantic segmentation task. Inspired by these works, we follow a multi-task learning schema, where semantic segmentation and depth estimation are jointly tackled in a Convolutional Neural Network (CNN). Since semantic segmenta- tion and depth estimation are two highly correlated tasks, approaching them jointly can be mutually beneficial. In this case, depth information along with the segmenta- tion annotation in the training data helps better defining the target of the training process of the classifier, instead of feeding the system blindly with an extra input channel. We design a novel hybrid CNN architecture by investigating the common attributes as well as the distinction for depth estimation and semantic segmentation. The proposed architecture is tested and compared with state-of-the-art approaches in different datasets.

Although outstanding results are achieved in semantic segmentation, the limita- tions in these approaches are also obvious. Semantic segmentation strongly relies on predefined semantics and a large amount of annotated data, which may not be avail- able in more general applications. On the other hand, classical image segmentation tackles the segmentation task in a more general way. But classical approaches hardly obtain object level segmentation due to the lack of higher level knowledge. Thus, in the second part of this thesis, we focus on an RGB-D based generic instance segmenta- tion problem where temporal information is available from the RGB-D video while no semantic information is provided. We present a novel generic segmentation approach for 3D point cloud video (stream data) thoroughly exploiting the explicit geometry iii and temporal correspondences in RGB-D. The proposed approach is validated and compared with state-of-the-art generic segmentation approaches in different datasets.

Finally, in the third part of this thesis, we present a method which combines the advantages in both semantic segmentation and generic segmentation, where we discover object instances using the generic approach and model them by learning from the few discovered examples by applying the approach of semantic segmentation. To do so, we employ the one shot learning technique, which performs knowledge transfer from a generally trained model to a specific instance model. The learned instance models generate robust features in distinguishing dierent instances, which is fed to the generic segmentation approach to perform improved segmentation. The approach is validated with experiments conducted on a carefully selected dataset.

}, url = {http://hdl.handle.net/10803/620762}, author = {X. Lin}, editor = {Casas, J. and M. Pard{\`a}s} } @inbook {bCampos, title = {Sentiment concept embedding for visual affect recognition}, booktitle = {Multimodal Behavior Analysis in theWild}, year = {2018}, publisher = {Elsevier}, organization = {Elsevier}, edition = {1}, chapter = {16}, abstract = {

Automatic sentiment and emotion understanding of general visual content has recently garnered much research attention. However, the large visual variance associated with\ high-level affective concepts presents a challenge when designing systems with high-performance requirements. One\ popular approach to bridge the {\textquotedblleft}affective gap{\textquotedblright} between\ low-level visual features and affective semantics consists of\ using Adjective Noun Pair (ANP) semantic constructs for\ concepts, e.g. {\textquotedblleft}beautiful landscape{\textquotedblright} or {\textquotedblleft}scary face{\textquotedblright} which\ act as a mid-level representation that can be recognized by\ visual classifers while still carrying an affective bias. In\ this work, we formulate the ANP detection task in images\ over a continuous space defined over an embedding that\ captures the inter-concept relationships between ANPs. We\ show how the compact representations obtained from the\ embeddeding extend the discrete concepts in the ontology\ and can be used for improved visual sentiment and emotion\ prediction, as well as new applications such as zero-shot\ ANP detection.

}, url = {https://www.elsevier.com/books/multimodal-behavior-analysis-in-the-wild/alameda-pineda/978-0-12-814601-9}, author = {V{\'\i}ctor Campos and Xavier Gir{\'o}-i-Nieto and Jou, Brendan and Jordi Torres and Chang, Shih-Fu} } @conference {cCasamitjanac, title = {Shared latent structures between imaging features and biomarkers in early stages of Alzheimer{\textquoteright}s disease}, booktitle = {Workshop on Predictive Intelligence in Medicine (PRIME), MICCAI}, year = {2018}, month = {2018}, address = {Granada, Spain}, abstract = {

In this work, we identify meaningful latent patterns in MR images for patients across the Alzheimer{\textquoteright}s disease (AD) continuum. For this purpose, we apply Projection to Latent Structures (PLS) method using cerebrospinal fluid (CSF) biomarkers (t-tau, p-tau, amyloid-beta) and age as response variables and imaging features as explanatory variables. Freesurfer pipeline is used to compute MRI surface and volumetric features\  \ resulting in 68 cortical ROIs and 84 cortical and subcortical ROIs, respectively. The main assumption of this work is that there are two main underlying processes governing brain morphology along the AD continuum: brain aging and dementia. We use two different and orthogonal PLS models to describe each process: PLS-aging and PLS-dementia. To define PLS-aging model we use normal aging subjects and age as predictor and response variables, respectively, while for PLS-dementia we only use demented subjects and biomarkers as response variables.\ 

}, author = {Adri{\`a} Casamitjana and Ver{\'o}nica Vilaplana and Paula Petrone and Jose Luis Molinuevo and Juan D. Gispert} } @inbook {bCasamitjana18a, title = {Shared Latent Structures Between Imaging Features and Biomarkers in Early Stages of Alzheimer{\textquoteright}s Disease}, booktitle = {PRedictive Intelligence in MEdicine}, volume = {11121}, year = {2018}, pages = {60-67}, publisher = {Springer International Publishing}, organization = {Springer International Publishing}, abstract = {

In this work, we identify meaningful latent patterns in MR images for patients across the Alzheimer{\textquoteright}s disease (AD) continuum. For this purpose, we apply Projection to Latent Structures (PLS) method using cerebrospinal fluid (CSF) biomarkers (t-tau, p-tau, amyloid-beta) and age as response variables and imaging features as explanatory variables. Freesurfer pipeline is used to compute MRI surface and volumetric features resulting in 68 cortical ROIs and 84 cortical and subcortical ROIs, respectively. The main assumption of this work is that there are two main underlying processes governing brain morphology along the AD continuum: brain aging and dementia. We use two different and orthogonal PLS models to describe each process: PLS-aging and PLS-dementia. To define PLS-aging model we use normal aging subjects and age as predictor and response variables, respectively, while for PLS-dementia we only use demented subjects and biomarkers as response variables.

}, issn = {978-3-030-00320-3}, doi = {10.1007/978-3-030-00320-3}, author = {Adri{\`a} Casamitjana and Ver{\'o}nica Vilaplana and Paula Petrone and Jose Luis Molinuevo and Juan D. Gispert} } @conference {cCampos18, title = {Skip RNN: Learning to Skip State Updates in Recurrent Neural Networks}, booktitle = {International Conference on Learning Representations (ICLR)}, year = {2018}, month = {01/2018}, abstract = {

Recurrent Neural Networks (RNNs) continue to show\  outstanding performance in sequence modeling tasks. However, training RNNs on long sequences often face challenges like slow inference, vanishing gradients and difficulty in capturing long term dependencies. In backpropagation through time settings, these issues are tightly coupled with the large, sequential computational graph resulting from unfolding the RNN in time. We introduce the Skip RNN model which extends existing RNN models by learning to skip state updates and shortens the effective size of the computational graph. This model can also be encouraged to perform fewer state updates through a budget constraint. We evaluate the proposed model on various tasks and show how it can reduce the number of required RNN updates while preserving, and sometimes even improving, the performance of the baseline RNN models.

Skip RNN: Learning to Skip State Updates in Recurrent Neural Networks from Xavier Giro-i-Nieto
}, author = {V{\'\i}ctor Campos and Jou, Brendan and Xavier Gir{\'o}-i-Nieto and Jordi Torres and Chang, Shih-Fu} } @conference {cCaminal18, title = {SLAM-based 3D outdoor reconstructions from LIDAR data}, booktitle = {IC3D}, year = {2018}, month = {12/2018}, publisher = {IEEE}, organization = {IEEE}, address = {Brussels, Belgium}, abstract = {

The use of depth (RGBD) cameras to reconstruct large outdoor environments is not feasible due to lighting conditions and low depth range. LIDAR sensors can be used instead. Most state of the art SLAM methods are devoted to indoor environments and depth (RGBD) cameras. We have adapted two SLAM systems to work with LIDAR data. Quantitative evaluations have been performed with LIDAR and RGBD data allowing to compare the systems. Results show that the best method with LIDAR is RTAB-Map with clear difference. Additionally, RTAB-Map has been used to create 3D reconstructions with and without the use of a visible color camera. This proves the potential of Lidar sensors for the reconstructionof outdoor environments for immersion or audiovisual production applications.

}, keywords = {3D imaging, Lidar cameras, mapping, point-cloud processing, SLAM, time-of-flight}, url = {http://www.stereopsia.com/international-conference-3d-immersion-ic3d}, author = {I. Caminal and Casas, J. and S. Royo} } @mastersthesis {xRoldana, title = {Speech-conditioned Face Generation with Deep Adversarial Networks}, year = {2018}, abstract = {

Image synthesis have been a trending task for the AI community in recent years. Many works have shown the potential of Generative Adversarial Networks (GANs) to deal with tasks such as text or audio to image synthesis. In particular, recent advances in deep learning using audio have inspired many works involving both visual and auditory information. In this work we propose a face synthesis method using audio and/or language representations as inputs. Furthermore, a dataset which relates speech utterances with a face and an identity has been built, fitting for other tasks apart from face synthesis such as speaker recognition or voice conversion.

Speech Conditioned Face Generation with Deep Adversarial Networks from Universitat Polit{\`e}cnica de Catalunya
}, author = {Rold{\'a}n, Francisco}, editor = {Pascual-deLaPuente, Santiago and Amaia Salvador and McGuinness, Kevin and Xavier Gir{\'o}-i-Nieto} } @conference {cPana, title = {SalGAN: Visual Saliency Prediction with Generative Adversarial Networks}, booktitle = {CVPR 2017 Scene Understanding Workshop (SUNw)}, year = {2017}, address = {Honolulu, Hawaii, USA}, abstract = {

We introduce SalGAN, a deep convolutional neural network for visual saliency prediction trained with adversarial examples. The first stage of the network consists of a generator model whose weights are learned by back-propagation computed from a binary cross entropy (BCE) loss over downsampled versions of the saliency maps. The resulting prediction is processed by a discriminator network trained to solve a binary classification task between the saliency maps generated by the generative stage and the ground truth ones. Our experiments show how adversarial training allows reaching state-of-the-art performance across different metrics when combined with a widely-used loss function like BCE.

}, url = {https://arxiv.org/abs/1701.01081}, author = {Pan, Junting and Cristian Canton-Ferrer and McGuinness, Kevin and O{\textquoteright}Connor, N. and Jordi Torres and Elisa Sayrol and Xavier Gir{\'o}-i-Nieto} } @conference {cAssens, title = {SaltiNet: Scan-path Prediction on 360 Degree Images using Saliency Volumes}, booktitle = {ICCV Workshop on Egocentric Perception, Interaction and Computing}, year = {2017}, month = {07/2017}, publisher = {IEEE}, organization = {IEEE}, address = {Venice, Italy}, abstract = {

We introduce SaltiNet, a deep neural network for scanpath prediction trained on 360-degree images. The first part of the network consists of a model trained to generate saliency volumes, whose parameters are learned by back-propagation computed from a binary cross entropy (BCE) loss over downsampled versions of the saliency volumes. Sampling strategies over these volumes are used to generate scanpaths over the 360-degree images. Our experiments show the advantages of using saliency volumes, and how they can be used for related tasks.

Winner of three awards at the Salient 360 Challenge at IEEE ICME 2017 (Hong Kong): Best Scan Path, Best Student Scan-path and Audience Award.

SaltiNet: The Temporal Dimension of Visual Attention Models from Xavier Giro-i-Nieto
}, isbn = {978-1-5386-1034-3}, doi = {10.1109/ICCVW.2017.275}, url = {http://ieeexplore.ieee.org/document/8265485/}, author = {Assens, Marc and McGuinness, Kevin and Xavier Gir{\'o}-i-Nieto and O{\textquoteright}Connor, N.} } @conference {cTorres, title = {Scaling a Convolutional Neural Network for classification of Adjective Noun Pairs with TensorFlow on GPU Clusters}, booktitle = {17th IEEE/ACM International Symposium on Cluster, Cloud and Grid Computing (CCGrid)}, year = {2017}, month = {05/2017}, publisher = {IEEE}, organization = {IEEE}, address = {Madrid, Spain}, abstract = {

Deep neural networks have gained popularity in recent years, obtaining outstanding results in a wide range of applications such as computer vision in both academia and multiple industry areas. The progress made in recent years cannot be understood without taking into account the technological advancements seen in key domains such as High Performance Computing, more specifically in the Graphic Processing Unit (GPU) domain. These kind of deep neural networks need massive amounts of data to effectively train the millions of parameters they contain, and this training can take up to days or weeks depending on the computer hardware we are using. In this work, we present how the training of a deep neural network can be parallelized on a distributed GPU cluster. The effect of distributing the training process is addressed from two different points of view. First, the scalability of the task and its performance in the distributed setting are analyzed. Second, the impact of distributed training methods on the training times and final accuracy of the models is studied. We used TensorFlow on top of the GPU cluster of servers with 2 K80 GPU cards, at Barcelona Supercomputing Center (BSC). The results show an improvement for both focused areas. On one hand, the experiments show promising results in order to train a neural network faster. The training time is decreased from 106 hours to 16 hours in our experiments. On the other hand we can observe how increasing the numbers of GPUs in one node rises the throughput, images per second, in a near-linear way. Morever an additional distributed speedup of 10.3 is achieved with 16 nodes taking as baseline the speedup of one node.

[CCGRID 2017] [Paper page]

}, url = {http://easychair.org/smart-program/CCGRID2017/2017-05-15.html$\#$session:13550}, author = {Jordi Torres and Sastre, Francesc and Yag{\"u}es, Maurici and V{\'\i}ctor Campos and Xavier Gir{\'o}-i-Nieto} } @conference {cBazazian16, title = {Segmentation-based Multi-Scale Edge Extraction to Measure the Persistence of Features in Unorganized Point Clouds}, booktitle = {International Conference on Computer Vision Theory and Applications}, year = {2017}, month = {02/2017}, address = {Porto, Portugal}, abstract = {

Edge extraction has attracted a lot of attention in computer vision. The accuracy of extracting edges in point clouds can be a significant asset for a variety of engineering scenarios. To address these issues, we propose a segmentation-based multi-scale edge extraction technique.\ 

In this approach, different regions of a point cloud are segmented by a global analysis according to the geodesic distance. Afterwards, a multi-scale operator is defined according to local neighborhoods. Thereupon, by applying this operator at multiple scales of the point cloud, the persistence of features is determined. We illustrate the proposed method by computing a feature weight that measures the likelihood of a point to be an edge, then detects the edge points based on that value at both global and local scales. Moreover, we evaluate quantitatively and qualitatively our method. Experimental results show that the proposed approach achieves a superior accuracy. Furthermore, we demonstrate the robustness of our approach in noisier real-world datasets.

}, doi = {10.5220/0006092503170325}, url = {http://www.scitepress.org/DigitalLibrary/Link.aspx?doi=10.5220\%2f0006092503170325}, author = {D. Bazazian and Casas, J. and Ruiz-Hidalgo, J.} } @conference {xLidona, title = {Semantic Summarization of Egocentric Photo Stream Events}, booktitle = {ACM Multimedia 2017 Workshop on Lifelogging Tools and Applications}, year = {2017}, month = {10/2017}, publisher = {ACM}, organization = {ACM}, address = {Mountain View, CA, USA}, abstract = {

With the rapid increase of users of wearable cameras in recent years and of the amount of data they produce, there is a strong need for automatic retrieval and summarization techniques. This work addresses the problem of automatically summarizing egocentric photo streams captured through a wearable camera by taking an image retrieval perspective. After removing non-informative images by a new CNN-based filter, \ images are ranked by relevance to ensure semantic diversity and \ finally re-ranked by a novelty criterion to reduce redundancy. \ To assess the results, a new evaluation metric is proposed which takes into account the non-uniqueness of the solution. Experimental results applied on a database of 7,110 images from 6 different subjects and evaluated by experts gave 95.74\% of experts satisfaction and a Mean Opinion Score of 4.57 out of 5.0.

}, doi = {10.1145/3133202.3133204}, url = {https://dl.acm.org/citation.cfm?id=3133204}, author = {Lidon, Aniol and Bola{\~n}os, Marc and Dimiccoli, M. and Radeva, Petia and Garolera, Maite and Xavier Gir{\'o}-i-Nieto} } @conference {cRomero-Lopez, title = {Skin Lesion Classification from Dermoscopic Images using Deep Learning}, booktitle = {The 13th IASTED International Conference on Biomedical Engineering (BioMed 2017)}, year = {2017}, month = {02/2017}, address = {Innsbruck Austria}, abstract = {

The recent emergence of deep learning methods for medical image analysis has enabled the development of intelligent medical imaging-based diagnosis systems that can assist the human expert in making better decisions about a patient{\textquoteright}s health. In this paper we focus on the problem of skin lesion classification, particularly early melanoma detection, and present a deep-learning based approach to solve the problem of classifying a dermoscopic image containing a skin lesion as malignant or benign. \ The proposed solution is built around the VGGNet convolutional neural network architecture and uses the transfer learning paradigm. Experimental results are encouraging: on the ISIC Archive dataset, the proposed method achieves a sensitivity value of 78.66\%, which is significantly higher than the current state of the art on that dataset.

[BioMed conference 2017]

}, keywords = {Convolutional Neural Networks, deep learning, machine learning, Medical Decision Support Systems, Medical Image Analysis, Skin Lesions}, url = {http://upcommons.upc.edu/handle/2117/103386}, author = {Romero-Lopez, Adria and Burdick, Jack and Xavier Gir{\'o}-i-Nieto and Marques, Oge} } @mastersthesis {xRomero-Lopez, title = {Skin Lesion Detection from Dermoscopic Images using Convolutional Neural Networks}, year = {2017}, abstract = {

Advisors: Oge Marques (Florida Atlantic University) and Xavier Giro-i-Nieto (UPC)

The recent emergence of machine learning and deep learning methods for medical image analysis has enabled the development of intelligent medical imaging-based diagnosis systems that can assist physicians in making better decisions about a patient{\textquoteright}s health. In particular, skin imaging is a field where these new methods can be applied with a high rate of success.\ 

This thesis focuses on the problem of automatic skin lesion detection, \ particularly on melanoma detection, by applying semantic segmentation and classification from dermoscopic images using a deep learning based approach.\ For the first problem, a U-Net convolutional neural network architecture is applied for an accurate extraction of the lesion region.\ For the second problem, the current model performs a binary classification (benign versus malignant) that can be used for early melanoma detection. The model is general enough to be extended to multi-class skin lesion classification. The proposed solution is built around the VGG-Net ConvNet architecture and uses the transfer learning paradigm.\ Finally, this work performs a comparative evaluation of classification \ alone (using the entire image) against a combination of the two approaches (segmentation followed by classification) in order to assess which of them achieves better classification results.

[Source code]

Skin Lesion Detection from Dermoscopic Images using Convolutional Neural Networks from Xavier Giro
}, author = {Romero-Lopez, Adria}, editor = {Xavier Gir{\'o}-i-Nieto and Marques, Oge} } @conference {cCampos, title = {Skip RNN: Learning to Skip State Updates in Recurrent Neural Networks}, booktitle = {NIPS Time Series Workshop 2017}, year = {2017}, month = {08/2017}, address = {Long Beach, CA, USA}, abstract = {

Recurrent Neural Networks (RNNs) continue to show \ outstanding performance in sequence modeling tasks. However, training RNNs on long sequences often face challenges like slow inference, vanishing gradients and difficulty in capturing long term dependencies. In backpropagation through time settings, these issues are tightly coupled with the large, sequential computational graph resulting from unfolding the RNN in time. We introduce the Skip RNN model which extends existing RNN models by learning to skip state updates and shortens the effective size of the computational graph. This model can also be encouraged to perform fewer state updates through a budget constraint. We evaluate the proposed model on various tasks and show how it can reduce the number of required RNN updates while preserving, and sometimes even improving, the performance of the baseline RNN models.

Skip RNN: Learning to Skip State Updates in Recurrent Neural Networks from Xavier Giro-i-Nieto
}, url = {https://imatge-upc.github.io/skiprnn-2017-telecombcn/}, author = {V{\'\i}ctor Campos and Jou, Brendan and Xavier Gir{\'o}-i-Nieto and Jordi Torres and Chang, Shih-Fu} } @article {xSalvador17a, title = {Snap a photo, get a recipe: pic2recipe uses AI to predict food ingredients}, year = {2017}, publisher = {Digital Trends}, abstract = {

No recipe, no problem. Using image recognition, we can now learn how to cook something just by analyzing a photo.

}, url = {https://www.digitaltrends.com/photography/pics2recipe-mit-research/}, author = {Amaia Salvador} } @conference {cLuque17, title = {Spatio-Temporal Road Detection from Aerial Imagery using CNNs}, booktitle = {International Conference on Computer Vision Theory and Applications}, year = {2017}, month = {2/2017}, address = {Porto, Portugal}, abstract = {

The main goal of this paper is to detect roads from aerial imagery recorded by drones. To achieve this, we propose a modification of SegNet, a deep fully convolutional neural network for image segmentation. In order to train this neural network, we have put together a database containing videos of roads from the point of view of a small commercial drone. Additionally, we have developed an image annotation tool based on the watershed technique, in order to perform a semi-automatic labeling of the videos in this database. The experimental results using our modified version of SegNet show a big improvement on the performance of the neural network when using aerial imagery, obtaining over 90\% accuracy.

}, doi = {10.5220/0006128904930500}, author = {Luque, B. and Morros, J.R. and Ruiz-Hidalgo, J.} } @phdthesis {dBellot17, title = {Study of Gene Regulatory Networks Inference Methods from Gene Expression Data}, year = {2017}, author = {Pau Bellot}, editor = {Salembier, P.} } @conference {cPan, title = {Shallow and Deep Convolutional Networks for Saliency Prediction}, booktitle = {IEEE Conference on Computer Vision and Pattern Recognition, CVPR}, year = {2016}, month = {06/2016}, publisher = {Computer Vision Foundation / IEEE}, organization = {Computer Vision Foundation / IEEE}, address = {Las Vegas, NV, USA}, abstract = {

The prediction of salient areas in images has been traditionally addressed with hand-crafted features based on neuroscience principles. This paper, however, addresses the problem with a completely data-driven approach by training a convolutional neural network (convnet). The learning process is formulated as a minimization of a loss function that measures the Euclidean distance of the predicted saliency map with the provided ground truth. The recent publication of large datasets of saliency prediction has provided enough data to train end-to-end architectures that are both fast and accurate. Two designs are proposed: a shallow convnet trained from scratch, and a another deeper solution whose first three layers are adapted from another network trained for classification. To the authors knowledge, these are the first end-to-end CNNs trained and tested for the purpose of saliency prediction.

}, url = {http://www.cv-foundation.org/openaccess/content_cvpr_2016/papers/Pan_Shallow_and_Deep_CVPR_2016_paper.pdf}, author = {Pan, Junting and McGuinness, Kevin and Elisa Sayrol and O{\textquoteright}Connor, N. and Xavier Gir{\'o}-i-Nieto} } @mastersthesis {xNespereira, title = {Siamese Convolutional Neural Network for Learning Object Similarities in RGB-D Images}, year = {2016}, abstract = {

Student: Alejandro Nespereira

Advisors: Farzad Husain (Catchoom), Tomasz Adamek (Catchoom) and Xavier Gir{\'o}-i-Nieto (UPC)

Program:\ Master in Computer Vision\ (Class of 2016)

This report explores the suitability of using a Siamese Convolutional Neural Network (CNN) for the task of false positive rejection. We present a Siamese CNN model trained with an in-house dataset of weakly textured objects. Our model is able to successfully assert the classification of an object detection pipeline with unseen new objects. Additionally, we also compare it with a hand-crafted method in order to compare its performance. We demonstrate the usage of our model by learning to discriminate between inter and intra object classes for a challenging dataset.

}, author = {Nespereira, Alejandro}, editor = {Husain, Farzad and Adamek, Tomasz and Xavier Gir{\'o}-i-Nieto} } @article {Pont-Tuset2015c, title = {Supervised Evaluation of Image Segmentation and Object Proposal Techniques}, journal = {IEEE Transactions on Pattern Analysis and Machine Intelligence (TPAMI)}, volume = {38}, year = {2016}, pages = {1465 - 1478}, author = {Jordi Pont-Tuset and Marqu{\'e}s, F.} } @article {aVilaplana, title = {Saliency Maps on Image Hierarchies}, journal = {Signal Processing: Image Communication. Special Issue on Recent Advances in Saliency Models, Applications and Evaluations}, volume = {38}, year = {2015}, pages = {84-99}, abstract = {

In this paper we propose two saliency models for salient object segmentation based on a hierarchical image segmentation, a tree-like structure that represents regions at di erent scales from the details to the whole image (e.g. gPb-UCM, BPT). The first model is based on a hierarchy of image partitions. The saliency at each level is computed on a region basis, taking into account the contrast between regions. The maps obtained for the di erent partitions are then integrated into a nal saliency map. The second model directly works on the structure created by the segmentation algorithm, computing saliency at each node and integrating these cues in a straightforward manner into a single saliency map. We show that the proposed models produce high quality saliency maps. Objective evaluation demonstrates that the two methods achieve state-of-the-art performance in several benchmark datasets.

}, issn = {0923-5965}, author = {Ver{\'o}nica Vilaplana} } @mastersthesis {xLidon, title = {Semantic and Diverse Summarization of Egocentric Photo Events}, year = {2015}, abstract = {

Student: Aniol Lidon

Advisors: Petia Radeva (UB) and Xavier Gir{\'o}-i-Nieto (UPC)

Program: Master in Computer Vision

Grade: A (9.8/10.0)

This project generates visual summaries of events depicted from egocentric photos taken with a wearable camera. These summaries are addressed to mild-dementia patients in order to exercise their memory in a daily base. The main contribution is an iterative approach that guarantees the semantic diversity of the summary and a novel soft metric to assess subjective results. Medical experts validated the proposed solution with a Mean Opinion Score of 4.6 out of of 5.0. The flexibility and quality of the solution was also tested in the 2015 Retrieving Diverse Social Images Task from the scientific international benchmark, MediaEval.\ 

Semantic and Diverse Summarization of Egocentric Photo Events from Xavier Giro
Aniol Lidon Master Thesis
}, author = {Lidon, Aniol}, editor = {Radeva, Petia and Xavier Gir{\'o}-i-Nieto} } @conference {cSalembier15, title = {Study of Binary Partition Tree Pruning Techniques for Polarimetric SAR images}, booktitle = {International Symposium on Mathematical Morphology, ISMM 2015}, year = {2015}, month = {05/2015}, publisher = {Springer}, organization = {Springer}, address = {Reykjavik, Iceland}, isbn = {978-3-319-18719-8}, author = {Salembier, P.} } @conference {Bell1512:Study, title = {Study of Normalization and Aggregation Approaches for Consensus Network Estimation}, booktitle = {2015 IEEE Symposium Series on Computational Intelligence: IEEE Symposium on Artificial Life (2015 IEEE ALIFE)}, year = {2015}, address = {Cape Town, South Africa}, abstract = {

Inferring gene regulatory networks from expression data is a very difficult problem that has raised the interest of the scientific community. Different algorithms have been proposed to try to solve this issue, but it has been shown that the different methods have some particular biases and strengths, and none of them is the best across all types of data and datasets. As a result, the idea of aggregating various network inferences through a consensus mechanism naturally arises. In this paper, a common framework to standardize already proposed consensus methods is presented, and based on this framework different proposals are introduced and analyzed in two different scenarios: Homogeneous and Heterogeneous. The first scenario reflects situations where the networks to be aggregated are rather similar because the are obtained with inference algorithms working on the same data, whereas the second scenario deals with very diverse networks because various sources of data are used to generate the individual networks. A procedure for combining multiple network inference algorithms is analyzed in a systematic way. The results show that there is a very significant difference between these two scenarios, and that the best way to combine networks in the Heterogeneous scenario is not the most commonly used. We show in particular that aggregation in the Heterogeneous scenario can be very beneficial if the individual networks are combined with our new proposed method ScaleLSum.

}, author = {P. Bellot and Salembier, P. and Albert Oliveras and Meyer, P. E.} } @phdthesis {dAlcoverro14, title = {Stochastic optimization and interactive machine learning for human motion analysis}, year = {2014}, month = {12/2014}, school = {UPC}, address = {Barcelona}, abstract = {

The analysis of human motion from visual data is a central issue in the computer vision research community as it enables a wide range of applications and it still remains a challenging problem when dealing with unconstrained scenarios and general conditions. Human motion analysis is used in the entertainment industry for movies or videogame production, in medical applications for rehabilitation or biomechanical studies. It is also used for human computer interaction in any kind of environment, and moreover, it is used for big data analysis from social networks such as Youtube or Flickr, to mention some of its use cases. In this thesis we have studied human motion analysis techniques with a focus on its application for smart room environments. That is, we have studied methods that will support the analysis of people behavior in the room, allowing interaction with computers in a natural manner and in general, methods that introduce computers in human activity environments to enable new kind of services but in an unobstrusive mode. The thesis is structured in two parts, where we study the problem of 3D pose estimation from multiple views and the recognition of gestures using range sensors. First, we propose a generic framework for hierarchically layered particle filtering (HPF) specially suited for motion capture tasks. Human motion capture problem generally involve tracking or optimization of high-dimensional state vectors where also one have to deal with multi-modal pdfs. HPF allow to overcome the problem by means of multiple passes through substate space variables. Then, based on the HPF framework, we propose a method to estimate the anthropometry of the subject, which at the end allows to obtain a human body model adjusted to the subject. Moreover, we introduce a new weighting function strategy for approximate partitioning of observations and a method that employs body part detections to improve particle propagation and weight evaluation, both integrated within the HPF framework. The second part of this thesis is centered in the detection of gestures, and we have focused the problem of reducing annotation and training efforts required to train a specific gesture. In order to reduce the efforts required to train a gesture detector, we propose a solution based on online random forests that allows training in real-time, while receiving new data in sequence. The main aspect that makes the solution effective is the method we propose to collect the hard negatives examples while training the forests. The method uses the detector trained up to the current frame to test on that frame, and then collects samples based on the response of the detector such that they will be more relevant for training. In this manner, training is more effective in terms of the number of annotated frames required.

}, url = {http://hdl.handle.net/10803/285337}, author = {Alcoverro, M.}, editor = {M. Pard{\`a}s and Casas, J.} } @conference {cVilaplana13, title = {Salient Object Detection on a Hierarchy of Image Partitions}, booktitle = {IEEE Int. Conf. in Image Processing, ICIP 2013}, year = {2013}, month = {09/2013}, address = {Melbourne, Australia}, author = {Ver{\'o}nica Vilaplana and G. Muntaner} } @article {aTochon13, title = {Segmentation hyperspectrales de forets tropicales par arbres de partition binaires}, journal = {Revue fran{\c c}aise de photogramm{\'e}trie et de t{\'e}l{\'e}d{\'e}tection}, volume = {202}, year = {2013}, month = {May 2013}, pages = {55-65}, author = {G. Tochon and J.B. Feret and Valero, S. and R.E. Martin and R. Tupayachi and Chanussot, J. and Salembier, P. and G. Asner} } @conference {cPont-Tuset12, title = {Supervised Assessment of Segmentation Hierarchies}, booktitle = {European Conference on Computer Vision (ECCV)}, year = {2012}, month = {01/2012}, abstract = {

This paper addresses the problem of the supervised assessment of hierarchical region-based image representations. Given the large amount of partitions represented in such structures, the supervised assessment approaches in\ the literature are based on selecting a reduced set of representative partitions and\ evaluating their quality. Assessment results, therefore, depend on the partition selection strategy used. Instead, we propose to find the partition in the tree that best\ matches the ground-truth partition, that is, the upper-bound partition selection.

We show that different partition selection algorithms can lead to different conclusions regarding the quality of the assessed trees and that the upper-bound partition\ selection provides the following advantages: 1) it does not limit the assessment\ to a reduced set of partitions, and 2) it better discriminates the random trees from\ actual ones, which reflects a better qualitative behavior. We model the problem as\ a Linear Fractional Combinatorial Optimization (LFCO) problem, which makes\ the upper-bound selection feasible and efficient.

[Poster]

}, doi = {10.1007/978-3-642-33765-9_58}, author = {Jordi Pont-Tuset and Marqu{\'e}s, F.} } @phdthesis {dSalvador11, title = {Surface Reconstruction for Multi-View Video}, year = {2011}, school = {Universitat Polit{\`e}cnica de Catalunya (UPC)}, type = {phd}, abstract = {

This thesis introduces a methodology for obtaining an alternative representation of video sequences captured by calibrated multi-camera systems in controlled environments with known scene background. This representation consists in a 3D description of the surfaces of foreground objects, which allows for the recovering of part of the 3D information of the original scene lost in the projection process in each camera.

The choice of the type of representation and the design of the reconstruction techniques are driven by three requirements that appear in smart rooms or recording studios. In these scenarios, video sequences captured by a multi-camera rig are used both for analysis applications and interactive visualization methods. The requirements are: the reconstruction method must be fast in order to be usable in interactive applications, the surface representation must provide a compression of the multi-view data redundancies and this representation must also provide all the relevant information to be used for analysis applications as well as for free-viewpoint video.

Once foreground and background are segregated for each view, the reconstruction process is divided in two stages. The first one obtains a sampling of the foreground surfaces (including orientation and texture), whereas the second provides closed, continuous surfaces from the samples, through interpolation.

The sampling process is interpreted as a search for 3D positions that result in feature matchings between different views. This search process can be driven by different mechanisms: an image-based approach, another one based on the deformation of a surface from frame to frame or a statistical sampling approach where samples are searched around the positions of other detected samples, which is the fastest and easiest to parallelize of the three approaches.

A meshing algorithm is also presented, which allows for the interpolation of surfaces between samples. Starting by an initial triangle, which connects three points coherently oriented, an iterative expansion of the surface over the complete set of samples takes place. The proposed method presents a very accurate reconstruction and results in a correct topology. Furthermore, it is fast enough to be used interactively.

The presented methodology for surface reconstruction permits obtaining a fast, compressed and complete representation of foreground elements in multi-view video, as reflected by the experimental results.

}, url = {http://hdl.handle.net/10803/108907}, author = {Salvador, J.}, editor = {Casas, J.} } @mastersthesis {xGirvent10, title = {Servei de v{\'\i}deos a la carta per a l{\textquoteright}iPhone}, year = {2010}, abstract = {

Els tel{\`e}fons intel{\textperiodcentered}ligents (smartphones) estan totalment integrats a la societat. Nombroses cadenes de televisi{\'o}, prove{\"\i}dores de continguts o portals webs desenvolupen aplicacions per a veure la televisi{\'o} en directe o reproduir, en temps real, v{\'\i}deos escollits a la carta. En aquest projecte es desenvolupa un servei de v{\'\i}deos a la carta per a iPhone, que permet reproduir v{\'\i}deos en temps real utilitzant el protocol HTTP Live Streaming d{\textquoteright}Apple. En aquesta mem{\`o}ria es detalla com configurar un servidor de v{\'\i}deo i el desenvolupament d{\textquoteright}una aplicaci{\'o} per a l{\textquoteright}iPhone que reprodueixi v{\'\i}deos en temps real. El servidor es programa i configura en Linux i s{\textquoteright}utilitza programari lliure per a l{\textquoteright}adaptaci{\'o} del v{\'\i}deo segons els requeriments de HTTP Live Streaming. Tamb{\'e} s{\textquoteright}emmagatzemen les metadades del v{\'\i}deo en una base de dades, a la qual s{\textquoteright}accedeix des de l{\textquoteright}iPhone per a mostrar-les en la interf{\'\i}cie. L{\textquoteright}aplicaci{\'o} per a l{\textquoteright}iPhone, permet navegar pel cat{\`a}leg de v{\'\i}deos, visualitzar les metadades del v{\'\i}deo com el t{\'\i}tol, la descripci{\'o} o una miniatura del v{\'\i}deo i reproduir-lo en temps real i en el dispositiu. Aquesta s{\textquoteright}ha desenvolupat en Mac, programant amb Objective-C i utilitzant les eines del kit de desenvolupament per a iPhone (iPhone SDK).

Servei de v{\'\i}deos a la carta per a l{\textquoteright}iPhone. from Xavi Gir{\'o}

}, keywords = {http, ios, iphone, streaming, video, web}, url = {http://hdl.handle.net/2099.1/13524}, author = {Bruna Girvent}, editor = {Xavier Gir{\'o}-i-Nieto} } @article {aHaro10, title = {Shape from incomplete silhouettes based on the reprojection error}, journal = {Image and vision computing}, volume = {28}, number = {9}, year = {2010}, pages = {1354{\textendash}1368}, issn = {0262-8856}, doi = {10.1016/j.imavis.2010.01.016}, url = {linkinghub.elsevier.com/retrieve/pii/S0262885610000326}, author = {Haro, G. and M. Pard{\`a}s} } @inbook {bAlcoverro10, title = {Skeleton and shape adjustment and tracking in multicamera environments}, booktitle = {Lecture notes in computer science}, volume = {6169/2010}, year = {2010}, pages = {88{\textendash}97}, publisher = {Springer}, organization = {Springer}, address = {Berlin / Heidelberg}, abstract = {

In this paper we present a method for automatic body model adjustment and motion tracking in multicamera environments. We introduce a set of shape deformation parameters based on linear blend skinning, that allow a deformation related to the scaling of the distinct bones of the body model skeleton, and a deformation in the radial direction of a bone. The adjustment of a generic body model to a specific subject is achieved by the estimation of those shape deformation parameters. This estimation combines a local optimization method and hierarchical particle filtering, and uses an efficient cost function based on foreground silhouettes using GPU. This estimation takes into account anthropometric constraints by using a rejection sampling method of propagation of particles. We propose a hierarchical particle filtering method for motion tracking using the adjusted model. We show accurate model adjustment and tracking for distinct subjects in a 5 cameras set up

}, isbn = {978-3-642-14060-0}, issn = {0302-9743}, doi = {10.1007/978-3-642-14061-7_9}, url = {http://www.springerlink.com/content/9662h2hwq81041g2/fulltext.pdf}, author = {Alcoverro, M. and Casas, J. and M. Pard{\`a}s} } @conference {cAlcoverro10, title = {Skeleton and shape adjustment and tracking in multicamera environments}, booktitle = {6th International Conference AMDO 2010}, year = {2010}, pages = {88{\textendash}97}, isbn = {978-3-642-14060-0}, doi = {10.1007/978-3-642-14061-7_9}, author = {Alcoverro, M. and Casas, J. and M. Pard{\`a}s} } @conference {cCuadras10, title = {Some measures of multivariate association relating two spectral data sets}, booktitle = {19th International Conference on Computational Statistics, COMSTAT 2010}, year = {2010}, address = {Paris, France}, author = {Cuadras, C. and Valero, S. and Salembier, P. and Chanussot, J.} } @conference {cCanton-Ferrer10, title = {Spatio-temporal alignment and hyperspherical radon transform for 3D gait recognition in multi-view environments}, booktitle = {2010 IEEE Computer Society Conference on Computer Vision and Pattern Recognition Workshops}, year = {2010}, pages = {116{\textendash}121}, isbn = {978-1-4244-7030-3/10}, doi = {10.1109/CVPRW.2010.5544615}, url = {http://ieeexplore.ieee.org/stamp/stamp.jsp?tp=\&arnumber=5544615}, author = {Cristian Canton-Ferrer and Casas, J. and M. Pard{\`a}s} } @conference {cSuau10, title = {Surface reconstruction by restricted and oriented propagation}, booktitle = {2010 IEEE International Conference on Image Processing}, year = {2010}, pages = {813{\textendash}816}, isbn = {952-15-1364-0}, doi = {10.1109/ICIP.2010.5652707}, author = {Suau, X. and Casas, J. and Ruiz-Hidalgo, J.} } @conference {cGiro10a, title = {System architecture of a web service for Content-Based Image Retrieval}, booktitle = {ACM International Conference On Image And Video Retrieval 2010}, year = {2010}, pages = {358{\textendash}365}, abstract = {

This paper presents the system architecture of a Content-Based Image Retrieval system implemented as a web service. The proposed solution is composed of two parts, a client running a graphical user interface for query formulation and a server where the search engine explores an image repository. The separation of the user interface and the search engine follows a Service as a Software (SaaS) model, a type of cloud computing design where a single core system is online and available to authorized clients. The proposed architecture follows the REST software architecture and HTTP protocol for communications, two solutions that combined with metadata coded in RDF, make the proposed system ready for its integration in the semantic web. User queries are formulated by visual examples through a graphical interface and content is remotely accessed also through HTTP communication. Visual descriptors and similarity measures implemented in this work are mostly defined in the MPEG-7 standard, while textual metadata is coded according to the Dublin Core specifications.

}, isbn = {978-1-4503-0117-6}, doi = {10.1145/1816041.1816093}, url = {http://doi.acm.org/10.1145/1816041.1816093}, author = {Xavier Gir{\'o}-i-Nieto and Ventura, C. and Jordi Pont-Tuset and Cort{\'e}s, S. and Marqu{\'e}s, F.} } @conference {cSalembier09, title = {Study on nonlocal morphological operators}, booktitle = {IEEE International Conference on Image Processing, ICIP 2009}, year = {2009}, address = {Cairo, Egypt}, author = {Salembier, P.} } @conference {cGallego08, title = {Segmentation and Tracking of Static and Moving Objects in Video Surveillance Scenarios}, booktitle = {IEEE International Conference on Image Processing}, year = {2008}, pages = {2716{\textendash}2719}, isbn = {1-4244-1764-3}, author = {Gallego, J. and M. Pard{\`a}s} } @article {aLandabaso08a, title = {Shape from inconsistent silhouette}, journal = {Computer vision and image understanding}, volume = {112}, number = {2}, year = {2008}, month = {11/2008}, pages = {210{\textendash}224}, abstract = {

Shape from silhouette (SfS) is the general term used to refer to the techniques that obtain a volume estimate from a set of binary images. In a first step, a number of images are taken from different positions around the scene of interest. Later, each image is segmented to produce binary masks, also called silhouettes, to delimit the objects of interest. Finally, the volume estimate is obtained as the maximal one which yields the silhouettes. The set of silhouettes is usually considered to be consistent which means that there exists at least one volume which completely explains them. However, silhouettes are normally inconsistent due to inaccurate calibration or erroneous silhouette extraction techniques. In spite of that, SfS techniques reconstruct only that part of the volume which projects consistently in all the silhouettes, leaving the rest unreconstructed. In this paper, we extend the idea of SfS to be used with sets of inconsistent silhouettes. We propose a fast technique for estimating that part of the volume which projects inconsistently and propose a criteria for classifying it by minimizing the probability of miss-classification taking into account the 2D error detection probabilities of the silhouettes. A number of theoretical and empirical results are given, showing that the proposed method reduces the reconstruction error.

}, issn = {1077-3142}, doi = {10.1016/j.cviu.2008.02.006}, author = {Landabaso, J. and M. Pard{\`a}s and Casas, J.} } @conference {cLandabaso08, title = {Shape from Inconsistent Silhouette for Free Viewpoint Video}, booktitle = {IEEE International Conference on Image Processing}, year = {2008}, pages = {213{\textendash}216}, isbn = {1-4244-1764-3}, author = {Landabaso, J. and Lizcano, L. and M. Pard{\`a}s} } @conference {cSalvador08, title = {Shape from Probability Maps with Image-Adapted Voxelization}, booktitle = {ECCV Workshop on Multi-camera and Multi-modal Sensor Fusion Algorithms and Applications (M2SFA2)}, year = {2008}, pages = {1{\textendash}12}, isbn = {0-7803-9076-8}, author = {Salvador, J. and Casas, J.} } @conference {cGiro-i-Nieto08, title = {System architecture for indexing regions in keyframes}, booktitle = {3rd International Conference on Semantic and digital Media Technologies}, year = {2008}, month = {12/2008}, pages = {1{\textendash}2}, address = {Koblenz, Germany}, abstract = {

This paper describes the design of an indexing system for a video database. The system uses region-based manual an- notations of keyframes to create models to automatically annotate new keyframes also at the region level. The pre- sented architecture includes user interfaces for training and querying the system, internal databases to manage ingested content and modelled semantic classes, as well as communi- cation interfaces to allow the system interconnection. The scheme is designed to work as a plug-in to an external Mul- timedia Asset Management (MAM) system.\ 

}, keywords = {architecture, Indexing, keyframe, MAM, region}, url = {http://hdl.handle.net/2117/10893}, author = {Xavier Gir{\'o}-i-Nieto and Marqu{\'e}s, F.} } @mastersthesis {xGil-Moreno07, title = {Sistema de gesti{\'o} de v{\'\i}deo off-line per una smart-room}, year = {2007}, keywords = {qt, recording, smartroom}, author = {A. Gil-Moreno}, editor = {Casas, J.} } @conference {cHaro07, title = {Stratification Learning: Detecting Mixed Density and Dimensionality in High Dimensional Point Clouds}, booktitle = {Neural Information Processing Systems NIPS}, year = {2007}, month = {12/2007}, publisher = {NIPS}, organization = {NIPS}, address = {Montreal}, abstract = {

The study of point cloud data sampled from a stratification, a collection of manifolds with possible different dimensions, is pursued in this paper. We present a technique for simultaneously soft clustering and estimating the mixed dimensionality and density of such structures. The framework is based on a maximum likelihood estimation of a Poisson mixture model. The presentation of the approach is completed with artificial and real examples demonstrating the importance of extending manifold learning to stratification learning.

}, author = {Haro, G. and Randall, Gregory and Sapiro, Guillermo} } @article {aSalembier07, title = {Structure description tools}, journal = {Journal of the American Society for Information Science}, volume = {58}, number = {9}, year = {2007}, pages = {1329{\textendash}1337}, issn = {0002-8231}, author = {Salembier, P. and Benitez, A.B} } @conference {cGiro-i-Nieto06c, title = {Solucions de programari lliure en un projecte d{\textquoteright}adaptaci{\'o} de dades a XML}, booktitle = {V Jornades de Programari Lliure de la UPC}, year = {2006}, author = {Xavier Gir{\'o}-i-Nieto and Camps, N.} } @phdthesis {dRuiz-Hidalgo06, title = {On the Synergy between indexing and compression representations for video sequences}, year = {2006}, school = {Universitat Polit{\`e}cnica de Catalunya (UPC)}, type = {phd}, author = {Ruiz-Hidalgo, J.}, editor = {Salembier, P.} } @conference {cQun05, title = {Shadow removal with blob-based Morphological Reconstruction for Error Corretion}, booktitle = {2005 IEEE International Conference on Acoustics, Speech, and Signal Processing}, year = {2005}, pages = {729{\textendash}732}, author = {Xu, Li-Qun and Landabaso, J. and M. Pard{\`a}s} } @conference {cCorrea05, title = {Silhouette-based Probabilistic 2D Human Motion Estimation for Real-Time Applications}, booktitle = {IEEE International Conference on Image Processing}, year = {2005}, pages = {836{\textendash}839}, isbn = {0-7803-9135-7}, author = {Correa, P. and Czyz, J. and Umeda, T. and Marqu{\'e}s, F. and Marichal, X. and Macq, B.} } @conference {cVilaplana05a, title = {Support Vector Data Description Based on PCA Features for Face Detection}, booktitle = {13th European Signal Processing Conference}, year = {2005}, isbn = {0-7803-6300-0}, author = {Ver{\'o}nica Vilaplana and Marqu{\'e}s, F.} } @conference {cCasas04, title = {Spatial-temporal Video Analysis for Improved Pedestrian Detection: Application to Density Estimation and Tracking in Demonstrations}, booktitle = {Image Analysis for Multimedia Interactive Services}, year = {2004}, pages = {334{\textendash}346}, isbn = {972-98115-7-1}, author = {Casas, J. and Puig-Sitjes, A. and Puig-Folch, P.} } @inbook {cGiro-i-Nieto03a, title = {Semantic Entity Detection Using Description Graphs}, booktitle = {Digital Media Processing for Multimedia Interactive Services}, year = {2003}, pages = {39{\textendash}42}, publisher = {World Scientific Publishing Co.}, organization = {World Scientific Publishing Co.}, address = {Singapore}, abstract = {

This paper presents a technique for the detection of Semantic Entities (SEs) in multimedia content. A definition of a SE in terms of lower-level SEs and their Relations (Rs) is proposed using Description Graphs (DGs). By analyzing the a/v information, an instance DG is built to be compared with a model DG of the SE. As a result, a confidence value is computed to express how well the SE is represented in the content. Examples of the use of this approach are presented in two different applications: detection of frontal faces and recognition of clusters of islands.

}, isbn = {981-238-355-7}, url = {http://books.google.es/books?id=vVvJINURimIC\&printsec=frontcover\&hl=ca\&source=gbs_ge_summary_r\&cad=0$\#$v=onepage\&q\&f=false}, author = {Xavier Gir{\'o}-i-Nieto and Marqu{\'e}s, F.} } @book {eSayrol02, title = {Senyals i sistemes anal{\`o}gics: una introducci{\'o} pr{\`a}ctica}, year = {2002}, isbn = {84-8301-610-9}, url = {http://www.edicionsupc.es}, author = {Elisa Sayrol and Gasull, A. and Moreno, A. and Salavedra, J. and Vallverdu, F.} } @inbook {bSalembier02, title = {Systems Architecture}, booktitle = {Introduction to the mpeg-7: multimedia content description interface}, year = {2002}, pages = {33{\textendash}42}, publisher = {Wiley}, organization = {Wiley}, edition = {B. S. Manjunath, P. Salembier, T. Sikora (Eds.)}, chapter = {3}, isbn = {0471486787}, author = {Avaro, O. and Salembier, P.} } @book {eSayrol01, title = {Senyals i sistemes anal{\`o}gics: una introducci{\'o} pr{\`a}ctica}, year = {2001}, isbn = {84-8301-510-2}, url = {www.edicionsupc.es}, author = {Elisa Sayrol and Gasull, A. and Moreno, A. and Salavedra, J. and Vallverdu, F.} } @conference {cMaziere00, title = {Segmentation and tracking of video objects for content-based video indexing}, booktitle = {IEEE International Conference on Multimedia and Expo, ICME{\textquoteright}2000}, year = {2000}, pages = {305{\textendash}309}, address = {New York City, NY, USA}, author = {Maziere, M. and Chassaing, F. and Garrido, L. and Salembier, P.} } @conference {cSayrol99a, title = {Simulaci{\'o}n digital de se{\~n}ales y sistemas anal{\'o}gicos}, booktitle = {III Congreso de Usuarios de MATLAB}, year = {1999}, pages = {67{\textendash}76}, isbn = {84-699-1358-1}, author = {Sayrol E. and Gasull, A. and Moreno, A. and Vallverdu, F. and Salavedra, J. and Albert Oliveras} } @conference {cRuiz-Hidalgo98, title = {The segmentation of images via scale-space trees}, booktitle = {British Machine Vision Conference}, year = {1998}, month = {09/1998}, pages = {33{\textendash}43}, address = {Southampton, UK}, abstract = {

A useful representation of an image would be an object tree in which nodes represent objects, or parts of objects, and which includes at least one node that, together with its children, represents each object: a grandmother node. It is shown that scale-trees, obtained from greyscale images, approximate such a tree. It is then shown how they may be modified using other attributes to more closely become object trees. The result is a data structure that provides {\textquotedblleft}handles{\textquotedblright} for every element of the image that can be used for manipulating the image. This segmentation has potential for object recognition.

}, author = {Bangham, J. and Ruiz-Hidalgo, J. and Harvey, R. and Cawley, G.} } @conference {cMorros97, title = {Scalable segmentation-based coding of video sequences addressing content-based functionalities}, booktitle = {IEEE International Conference on Image Processing}, year = {1997}, month = {10/1997}, pages = {1-4}, publisher = {IEEE}, organization = {IEEE}, address = {Santa Barbara, USA}, abstract = {

In this paper, we\ addressvideo\ scalability in the framework\ of\ a region-basedcoding\ system, allowingcontent-basedfunctionalities. The proposed algorithm can construct scaled layers from a\ videosequence, each one with either fixed bit-rate or fixed quality, allowing\ contentbased\ manipulation. Two modes\ of\ operation have been defined: a supervised mode, that allows the user to select the objects to be\ coded\ in the enhancement layer, and an unsupervised mode, where this selection is done by the algorithm itself

}, doi = {10.1109/ICIP.1997.638658}, author = {Morros, J.R. and Marqu{\'e}s, F.} } @article {aMarcotegui97, title = {Segmentation of video sequences and rate control}, journal = {Annales des t{\'e}lecommunications. Annals of telecommunications}, volume = {52}, number = {7-8}, year = {1997}, pages = {380{\textendash}388}, issn = {0003-4347}, author = {Marcotegui, B. and Marqu{\'e}s, F. and Morros, J.R. and M. Pard{\`a}s and Salembier, P.} } @article {aPardas97a, title = {Segmentation of video sequences for partition tree generation}, journal = {Annales des t{\'e}lecommunications. Annals of telecommunications}, volume = {52}, number = {7-8}, year = {1997}, pages = {389{\textendash}396}, issn = {0003-4347}, author = {M. Pard{\`a}s and Salembier, P.} } @article {aSalembier97, title = {Segmentation-based video coding system allowing the manipulation of objects}, journal = {IEEE transactions on circuits and systems for video technology}, volume = {7}, number = {1}, year = {1997}, pages = {60{\textendash}74}, issn = {1051-8215}, doi = {10.1109/76.554418}, author = {Salembier, P. and Marqu{\'e}s, F. and M. Pard{\`a}s and Morros, J.R. and Corset, I. and Jeannin, S. and Bouchard, L. and Meyer, F. and Marcotegui, B.} } @conference {cOliveras97, title = {Stereo image analyis using connected operators}, booktitle = {IEEE International Conference on Acoustics, Speech, and Signal Processing, ICASSP 1997}, year = {1997}, pages = {3169{\textendash}3172}, address = {Munich, Germany}, isbn = {978-989-8109-05-7}, author = {Albert Oliveras and Garrido, L. and Salembier, P.} } @article {aTorres97, title = {Stochastic vector quantization of images}, journal = {Signal Processing}, volume = {62}, number = {3}, year = {1997}, month = {11/1997}, pages = {291{\textendash}301}, abstract = {

One of the most important steps in the vector quantization of images is the design of the codebook. The codebook is generally designed using the LBG algorithm, that is in essence a clustering algorithm which uses a large training set of empirical data that is statistically representative of the image to be quantized. The LBG algorithm, although quite effective for practical applications, is computationally very expensive and the resulting codebook has to be recalculated each time the type of image to be encoded changes. One alternative to the generation of the codebook, called stochastic vector quantization, is presented in this paper. Stochastic vector quantization (SVQ) is based on the generation of the codebook according to some previous model defined for the image to be encoded. The well-known AR model has been used to model the image in the current implementations of the technique, and has shown good performance in the overall scheme. To show the merit of the technique in different contexts, stochastic vector quantization is discussed and applied to both pixel-based and segmentation-based image coding schemes.

}, issn = {0165-1684}, doi = {10.1016/S0165-1684(97)00130-8}, author = {Torres, L. and Casas, J. and Arias, E.} } @conference {cMarques96, title = {A segmentation-based coding system allowing manipulation of objects}, booktitle = {IEEE International Conference on Image Processing, ICIP 1996}, year = {1996}, pages = {145{\textendash}175}, address = {Lausanne, Switzerland}, abstract = {

We present a coding scheme that achieves, for each image in the sequence, the best segmentation in terms of rate-distortion theory. It is obtained from a set of initial regions and a set of available coding techniques. The segmentation combines spatial and motion criteria. It selects at each area of the image the most adequate criterion for defining a partition in order to obtain the best compromise between cost and quality. In addition, the proposed scheme is very suitable for addressing content-based functionalities.

}, isbn = {0-7803-3258-X}, doi = {10.1109/ICIP.1996.560741}, author = {Marqu{\'e}s, F. and Salembier, P. and M. Pard{\`a}s and Morros, J.R. and Corset, I. and Jeannin, S. and Marcotegui, B. and Meyer, F.} } @conference {cMarques96b, title = {A segmentation-based coding system allowing manipulation of objects (sesame)}, booktitle = {IEEE INTERNATIONAL CONFERENCE ON IMAGE PROCESSING}, year = {1996}, author = {Marqu{\'e}s, F.} } @conference {cSalembier96c, title = {Segmentation-based video coding: temporal links and rate control}, booktitle = {VIII European Signal Processing Conference, EUSIPCO{\textquoteright}96}, year = {1996}, pages = {455{\textendash}458}, address = {Trieste, Italy}, author = {Salembier, P. and Marqu{\'e}s, F. and M. Pard{\`a}s} } @article {pSalembier96a, title = {Segmented Picture Coding Method and System and Corresponding Decoding Method and System}, number = {964009161-}, year = {1996}, type = {Invention patent}, author = {Salembier, P. and Marqu{\'e}s, F. and Corset, I. and Bouchard, L. and Jeannin, S. and M. Pard{\`a}s and Morros, J.R. and Meyer, F. and Marcotegui, B.} } @conference {cCasas96a, title = {Strong edge features for image coding}, booktitle = {International Symposium on Mathematical Morphology and its Applications to Image and Signal Processing III}, year = {1996}, pages = {443{\textendash}450}, isbn = {0-7923-9733-9}, author = {Casas, J. and Torres, L.} } @inbook {bCasas96c, title = {Strong edge features for image coding}, booktitle = {Mathematical Morphology and its Applications to Image and Signal Processing}, volume = {5}, year = {1996}, pages = {443{\textendash}450}, publisher = {Springer}, organization = {Springer}, address = {Boston}, abstract = {

A two component model is proposed for perceptual image coding For the first component of the model the watershed operator is used to detect strong edge features Then an efficientmorphological interpolation algorithm reconstructs the smooth areas of the image from the extracted edge information also known as sketch data The residual component containing ne textures is separately coded by a subband coding scheme The morphological operators involved in the coding of the primary component perform very e ciently compared to conventional techniques like the LGO operator used for the edge extraction or the di usion lters iteratively applied for the interpolation of smooth areas in previously reported sketch based coding schemes.

}, isbn = {0-793-9733-9}, doi = {10.1007/978-1-4613-0469-2_52}, author = {Casas, J. and Torres, L.}, editor = {Schafer, R. W. and Maragos, P. and Butt, M. A.} } @phdthesis {dPardas95, title = {Segmentaci{\'o}n Morfol{\'o}gica de Secuencias de Im{\'a}genes: Aplicaci{\'o}n a la Codificaci{\'o}n}, year = {1995}, school = {Universitat Polit{\`e}cnica de Catalunya (UPC)}, type = {phd}, author = {M. Pard{\`a}s}, editor = {Salembier, P.} } @conference {cTorres94, title = {Segmentation based coding of textures using stochastic vector quantization}, booktitle = {IEEE International Conference on Acoustics, Speech and Signal Processing}, year = {1994}, pages = {553{\textendash}556}, abstract = {

In second generation image compression techniques the image to be compressed is first segmented. The pixels are divided into mutually exclusive spatial regions based on some criteria. After segmentation, the image consists of regions separated by contours. Then, the information is coded describing the shapes and interiors of the regions. The interiors of the regions are usually encoded using polynomials. The objective of this paper is to encode the interior of the regions by stochastic vector quantization techniques. If the segmentation process has been well defined and the obtained regions are homogeneous, then it is possible to design a specific codebook suited to the statistics of each region. The approach is to design the codebook according to some previously defined model for the regions of the image found in the segmentation process. If the approach is combined with efficient contour coding techniques, good visual results for high compression rates are obtained

}, isbn = {84-600-7766-7}, doi = {10.1109/ICASSP.1994.389380}, author = {Torres, L. and Casas, J. and Diego, D.} } @conference {cSalembier94 , title = {Self-referred texture coding for segmentation-based codec}, booktitle = {SPIE Visual Communication and Image Processing, VCIP{\textquoteright}94}, year = {1994}, address = {Chicago (IL), USA}, author = {Salembier, P. and Lopez, F} } @conference {cMarques93, title = {Shape and location coding for contour images}, booktitle = {PROC. OF THE 1993 PICTURE CODING SYMPOSIUM}, year = {1993}, pages = {61{\textendash}63}, author = {Marqu{\'e}s, F. and Gasull, A.} } @book {eTorres93, title = {Sistemas Anal{\'o}gicos y Digitales de Televisi{\'o}n}, year = {1993}, publisher = {Edicions UPC}, organization = {Edicions UPC}, address = {Barcelona}, isbn = {84-7653-288-1}, author = {Torres, L. and Lleida, E. and Casas, J.} } @conference {cMarques93b, title = {Stochastic image model for segmentation. application to image coding}, booktitle = {SCANDINAVIAN CONFERENCE ON IMAGE ANALYSIS}, year = {1993}, pages = {265{\textendash}272}, author = {Marqu{\'e}s, F. and Gasull, A.} } @conference {cMarques92c, title = {Segmentacion de imagenes multiespectrales con tecnicas piramidales}, booktitle = {VII Simposium Nacional de la Uni{\'o}n Cient{\'\i}fica Internacional de Radio}, year = {1992}, pages = {372{\textendash}376}, author = {Marqu{\'e}s, F. and Gasull, A.} } @conference {cMarques92a, title = {Segmentacion no supervisada de imagenes mediante campos aleatorios}, booktitle = {SIMPOSIUM NACIONAL DE RECONOZIMIENTO DE FORMAS Y ANALISIS IMAGENES}, year = {1992}, pages = {55{\textendash}62}, isbn = {3-8007-2300-X}, author = {Marqu{\'e}s, F. and Gasull, A.} } @article {aSalembier92a, title = {Size-sensitive multiresolution decomposition of images with rank..}, journal = {Signal processing}, volume = {27}, number = {2}, year = {1992}, pages = {205{\textendash}241}, issn = {0165-1684}, author = {Salembier, P.} } @article {aSalembier92, title = {Structuring element adaptation for morphological filters}, journal = {Journal of visual communication and image representation}, volume = {3}, number = {2}, year = {1992}, pages = {115{\textendash}136}, issn = {1047-3203}, author = {Salembier, P.} } @conference {cMarques91a, title = {Segmentacion de imagenes mediante modelos de gibbs-markov}, booktitle = {VI Simposium Nacional de la Uni{\'o}n Cient{\'\i}fica Internacional de Radio}, year = {1991}, pages = {975{\textendash}979}, author = {Marqu{\'e}s, F. and Gasull, A.} } @conference {cNadeu90, title = {Simbad:a tool for speech analysis and synthesis}, booktitle = {IASTED INT.CONF.SIGNAL PROC.\&DIG.FILT.}, year = {1990}, isbn = {1522-4880}, author = {Nadeu, C. and Albert Oliveras and Mari{\~n}o, J.} }