@inbook {bCalm25, title = {Identifying brain ageing trajectories using variational autoencoders}, booktitle = {PRedictive Intelligence in MEdicine}, volume = {15155}, number = {Lecture Notes in Computer Sciences}, year = {2025}, publisher = {Springer International Publishing}, organization = {Springer International Publishing}, author = {Berta Calm and Irene Cumplido-Mayoral and Juan Domingo Gispert and Ver{\'o}nica Vilaplana} } @conference {cCalm, title = {Identifying brain ageing trajectories using variational autoencoders with regression model in neuroimaging data stratified by sex and validated against dementia-related risk factors}, booktitle = {7th International Workshop on PRedictive Intelligence in MEdicine, MICCAI 2024}, year = {2024}, month = {10/2024}, abstract = {

Neuroimaging-derived brain age has been identified as a pro\-mising biomarker for accelerated brain age; however, the ageing process is highly heterogeneous and there is a need to further study the different brain ageing trajectories. In this study, we implemented a variational autoencoder (VAE) based model coupled with regression to identify different age-related patterns. Additionally, we correlated the patterns obtained, using a linear regression approach, with dementia-related risk factors. The model was evaluated in different cohorts, UK Biobank and ALFA+, to assess the robustness of the approach. The results showed a feasible strategy for detecting and validating brain age-related trajectories to identify possible early deviations using morphological brain data.

}, author = {Berta Calm and Irene Cumplido-Mayoral and Juan Domingo Gispert and Ver{\'o}nica Vilaplana} } @phdthesis {dHernandez24, title = {Implementation of personalized medicine in malignant melanoma patients aided by artificial intelligence}, volume = {PhD}, year = {2024}, month = {11/2024}, type = {PhD Thesis}, abstract = {

Skin cancer, characterized by the uncontrolled growth and spread of abnormal skin cells, remains a significant global health challenge. It predominantly appears on sun-exposed areas like the face, neck, and arms, though it can also develop in less exposed regions. Melanoma, the most aggressive form of skin cancer, is particularly concerning due to its rapid metastasis if not detected early. This aggressive nature underscores the need for timely diagnosis and treatment. While advances have been made in understanding skin cancer mechanisms, the extensive surface area of the skin, as the body{\textquoteright}s largest organ, makes it easy for lesions to go undetected. However, the advent of prospective databases of clinical and imaging data has equipped the medical community with highly sensitive and specific biomarkers for cancer diagnosis. These biomarkers facilitate early detection and accurate diagnosis through non-invasive methods, allowing for effective treatment of melanoma before it spreads. This thesis introduces tools for analyzing melanoma using clinical data and Whole Slide Imaging (WSI). It focuses on identifying early-stage melanoma patients through risk grouping and biomarker detection. By applying advanced survival analysis, pattern recognition techniques, and statistical clustering, we develop predictive and interpretability models aimed at improving early detection and diagnosis. The main outcomes of this thesis are five-fold. Firstly, we evaluate various survival analysis algorithms on cutaneous melanoma datasets, identifying the most effective methods. Our analysis reveals that, to date, treebased methods still surpass deep learning models in performance on survival analysis datasets Secondly, we introduce a Python library, SurvLIMEpy, to enhance model explainability for time-to-event data, demonstrating its validity through experiments with both simulated and large-scale melanoma datasets. Thirdly, our analysis of feature importance in the trained models reveals that the features identified by SurvLIMEpy align closely with clinically relevant features reported in medical literature. Fourthly, we show that machine learning models for patient stratification outperform the AJCC staging system when combined with Quantile-Based Survival Clustering. Finally, we explore the use of AI to predict biomarker status from WSIs, utilizing selfsupervised feature extractors and Multiple Instance Learning. The concluding sections discuss the effectiveness of the methodologies employed, their benefits, and their potential impact on the treatment of skin cancer patients while also outlining future research directions to address this critical medical and scientific challenge.

}, doi = {10.5821/dissertation-2117-417796 }, author = {Carlos Hernandez}, editor = {Ver{\'o}nica Vilaplana} } @inbook {bHernandez, title = {Interpreting Machine Learning Models for Survival Analysis: A Study of Cutaneous Melanoma Using the SEER Database}, booktitle = {Explainable Artificial Intelligence and Process Mining Applications for Healthcare,}, volume = {2020}, number = {Communications in Computer and Information Science}, year = {2024}, publisher = {Springer, Cham}, organization = {Springer, Cham}, abstract = {

In this study, we train and compare three types of machine learning algorithms for Survival Analysis: Random Survival Forest, DeepSurv and DeepHit, using the SEER database to model cutaneous malignant melanoma. Additionally, we employ SurvLIMEpy library, a Python package designed to provide explainability for survival machine learning models, to analyse feature importance. The results demonstrate that machine learning algorithms outperform the Cox Proportional Hazards Model. Our work underscores the importance of explainability methods for interpreting black-box models and provides insights into important features related to melanoma prognosis.

}, isbn = {978-3-031-54302-9}, issn = {978-3-031-54303-6}, doi = {https://doi.org/10.1007/978-3-031-54303-6_6}, author = {Carlos Hernandez and Cristian Pach{\'o}n-Garc{\'\i}a and Pedro Delicado and Ver{\'o}nica Vilaplana} } @conference {cHernandez23, title = {Interpreting Machine Learning models for Survival Analysis: A study of Cutaneous Melanoma using the SEER Database}, booktitle = {XAI-Healthcare 2023 Workshop at 21st International Conference of Artificial Intelligence in Medicine (AIME 2023)}, year = {2023}, month = {06/2023}, address = {Portoroz, Slovenia}, author = {Carlos Hernandez and Cristian Pach{\'o}n-Garc{\'\i}a and Pedro Delicado and Ver{\'o}nica Vilaplana} } @phdthesis {dBellver, title = {Image and Video Object Segmentation in Low Supervision Scenarios}, volume = {PhD}, year = {2021}, month = {03/2021}, school = {Universitat Politecnica de Catalunya}, type = {Academic}, address = {Barcelona}, abstract = {

Image and video segmentation are central tasks within the computer vision field. Nevertheless, deep learning solutions for segmentation typically rely on pixel-level annotations, which are very costly to collect. Likewise, some segmentation systems require human interaction at inference time, which involves effort for the end-user. In this thesis, we look into diverse supervision scenarios for image and video object segmentation. We discern between supervision when learning the model, i.e., which type of annotations are used during training, and supervision at inference, namely which kind of human input is required when running the system. Our target are models that require low forms of supervision.

In the first part of the thesis we present a novel recurrent architecture for video object segmentation that is end-to-end trainable in a fully-supervised setup, and that does not require any post-processing step, i.e., the output of the model directly solves the addressed task. The second part of the thesis aims at lowering the annotation cost, in terms of labeling time, needed to train image segmentation models. We explore semi-supervised pipelines and show results when a very limited budget is available. The third part of the dissertation attempts to alleviate the supervision required by semi-automatic systems at inference time. Particularly, we focus on semi-supervised video object segmentation, which typically requires generating a binary mask for each instance to be tracked. In contrast, we present a model for language-guided video object segmentation, which identifies the object to segment with a natural language expression. We study current benchmarks, propose a novel categorization of referring expressions for video, and identify the main challenges posed by the video task.

Evaluation committee: Zeynep Akata (University of T{\"u}bingen), Francesc Moreno-Noguer (UPC IRI-CSIC) and Yannis Kalantidis (Naver Labs Europ).

M{\'\i}riam Bellver Phd defense UPC 2021
}, author = {M{\'\i}riam Bellver}, editor = {Jordi Torres and Xavier Gir{\'o}-i-Nieto} } @conference {cHernandez, title = {Implementation of personalized medicine in cutaneous melanoma patients aided by artificial intelligence}, booktitle = {10th World Congress of2 Melanoma / 17th EADO Congress}, year = {2021}, month = {04/2021}, author = {Carlos Hernandez and Anil Kiroglu and Sergio Garc{\'\i}a and Joan Ficapal and Julio Burgos and Sebastian Podlipnik and Neus Calbet and Susana Puig and Josep Malvehy and Ver{\'o}nica Vilaplana and Marc Combalia} } @unpublished {xBonet21, title = {Improved Neural Network Generalization using Channel-Wise NNK Graph Constructions}, year = {2021}, publisher = {Final Year Project, UPC}, abstract = {

State-of-the-art neural network architectures continue to scale in size and deliver impressive results on unseen data points at the expense of poor interpretability. In the deep layers of these models we often encounter very high dimensional feature spaces, where constructing graphs from intermediate data representations can lead to the well-known curse of dimensionality. We propose a channel-wise graph construction method that works on lower dimensional subspaces and provides a new channel-based perspective that leads to better interpretability of the data and relationship between channels. In addition, we introduce a novel generalization estimate based on the proposed graph construction method with which we perform local polytope interpolation. We show its potential to replace the standard generalization estimate based on validation set performance to perform progressive channel-wise early stopping without requiring a validation set.

}, author = {Bonet, David}, editor = {Ortega, Antonio and Ruiz-Hidalgo, J. and Shekkizhar, Sarath} } @article {xGiro-i-Nieto20b, title = {Image and Video Object Segmentation with Low Supervision}, year = {2020}, abstract = {

Invited talk at VI-Lab, University oif Bristol (November 3rd, 2020 - 2pm)

Image and video segmentation are central tasks within the computer vision field. Nevertheless, deep learning solutions for segmentation typically rely on pixel-level annotations, which are very costly to collect. Likewise, some segmentation systems require human interaction at inference time, which involves some effort for the end-user. In this talk, we look into diverse supervision scenarios for image and video object segmentation. We discern between supervision when learning the model, i.e., which type of annotations are used during training, and supervision at inference, namely which kind of human input is required when running the system. Our target are models that require low forms of supervision

In the first part of the talk we present recurrent architectures for image and video object segmentation that are end-to-end trainable in a fully-supervised setup, and that does not require any post-processing step, i.e., the output of the model directly solves the addressed task. The second part of the talk aims at lowering the annotation cost, in terms of labelling time, needed to train image segmentation models. We explore semi-supervised pipelines and show results when a very limited budget is available. The third part of the session attempts to alleviate the supervision required by semi-automatic systems at inference time. Particularly, we focus on semi-supervised video object segmentation, which typically requires generating a binary mask for each instance to be tracked. In contrast, we present a model for language-guided video object segmentation, which identifies the object to segment with a natural language expression. We study current benchmarks, propose a novel categorization of referring expressions for video, and propose a method to generate synthetic referring expressions.

Slides

}, author = {Xavier Gir{\'o}-i-Nieto} } @article {xGiro-i-Nieto20a, title = {Image Segmentation with Deep Learning}, year = {2020}, abstract = {

Image segmentation is a classic computer vision task that aims at labeling pixels with semantic classes. These slides provide an overview of the basic approaches applied from the deep learning field to tackle this challenge and presents the basic subtasks (semantic, instance and panoptic segmentation) and related datasets.

Presented at the International Summer School on Deep Learning (ISSonDL) 2020 held online and organized by the University of Gdansk (Poland) between the 30th August and 2nd September.

Image Segmentation with Deep Learning - Xavier Giro \& Carles Ventura - ISSonDL 2020 from Universitat Polit{\`e}cnica de Catalunya
}, author = {Xavier Gir{\'o}-i-Nieto and Ventura, C.} } @mastersthesis {xCaselles, title = {Integrating low-level motion cues in deep video saliency}, year = {2019}, abstract = {

This thesis investigates the importance of motion when predicting saliency in videos. Naturally, humans observe both dynamic and static objects. When we are focused on watching a video, we tend to keep our eyes on the objects that are moving in the scene, items that we quickly recognize, as well as to those that attract our attention. In this work, different experiments are presented to corroborate this implication. Various approaches will be shown implementing an adaptation of the SalBCE neural network by using only motion. A simple implementation is proposed for the generation of saliency maps using previously extracted static and dynamic information from the images. The DHF1K dataset has been used for the experiment{\textquoteright}s realization.

}, keywords = {Motion, Saliency, video}, author = {Caselles, Pol}, editor = {McGuinness, Kevin and Xavier Gir{\'o}-i-Nieto} } @mastersthesis {xDomingo19, title = {Interpretability of Deep Learning Models}, year = {2019}, abstract = {

In recent years we have seen growth on interest for Deep Learning (DL) algorithms on a variety of problems, due to their outstanding performance. This is more palpable on a multitude of elds, where self learning algorithms are becoming indispensable tools to help professionals solve complex problems.

However as these models are getting better, they also tend to be more complex and are sometimes referred to as "Black Boxes". The lack of explanations for the resulting predictions and the inability of humans to understand those decisions seems problematic.

In this project, dierent methods to increase the interpretability of Deep Neural Networks (DNN) such as Convolutional Neural Network (CNN) are studied. Additionally, we evaluate how these interpretability methods or techniques can be implemented, measured and applied to real-world problems, by creating a python ToolBox.

}, author = {Pau Domingo}, editor = {Ver{\'o}nica Vilaplana} } @conference {cSalvadorf, title = {Inverse Cooking: Recipe Generation from Food Images}, booktitle = {CVPR}, year = {2019}, month = {06/2019}, publisher = {OpenCVF / IEEE}, organization = {OpenCVF / IEEE}, address = {Long Beach, CA, USA}, abstract = {

People enjoy food photography because they appreciate food. Behind each meal there is a story described in a complex recipe and, unfortunately, by simply looking at a food image we do not have access to its preparation process. Therefore, in this paper we introduce an inverse cooking system that recreates cooking recipes given food images. Our system predicts ingredients as sets by means of a novel architecture, modeling their dependencies without imposing any order, and then generates cooking instructions by attending to both image and its inferred ingredients simultaneously. We extensively evaluate the whole system on the large-scale Recipe1M dataset and show that (1) we improve performance w.r.t. previous baselines for ingredient prediction; (2) we are able to obtain high quality recipes by leveraging both image and ingredients; (3) our system is able to produce more compelling recipes than retrieval-based approaches according to human judgment.

}, url = {http://openaccess.thecvf.com/content_CVPR_2019/html/Salvador_Inverse_Cooking_Recipe_Generation_From_Food_Images_CVPR_2019_paper.html}, author = {Amaia Salvador and Drozdzal, Michal and Xavier Gir{\'o}-i-Nieto and Romero, Adriana} } @conference {cBakas19, title = {Identifying the best machine learning algorithms for brain tumor segmentation, progression assessment, and overall survival prediction in the BRATS challenge}, booktitle = {MICCAI - Multimodal Brain Tumor Segmentation Challenge}, year = {2018}, month = {12/2018}, abstract = {

Gliomas are the most common primary brain malignancies, with different degrees of aggressiveness, variable prognosis and various heterogeneous histologic sub-regions, i.e., peritumoral edematous/invaded tissue, necrotic core, active and non-enhancing core. This intrinsic heterogeneity is also portrayed in their radio-phenotype, as their sub-regions are depicted by varying intensity profiles disseminated across multi-parametric magnetic resonance imaging (mpMRI) scans, reflecting varying biological properties. Their heterogeneous shape, extent, and location are some of the factors that make these tumors difficult to resect, and in some cases inoperable. The amount of resected tumor is a factor also considered in longitudinal scans, when evaluating the apparent tumor for potential diagnosis of progression. Furthermore, there is mounting evidence that accurate segmentation of the various tumor sub-regions can offer the basis for quantitative image analysis towards prediction of patient overall survival. This study assesses the state-of-the-art machine learning (ML) methods used for brain tumor image analysis in mpMRI scans, during the last seven instances of the International Brain Tumor Segmentation (BraTS) challenge, i.e., 2012-2018. Specifically, we focus on i) evaluating segmentations of the various glioma sub-regions in pre-operative mpMRI scans, ii) assessing potential tumor progression by virtue of longitudinal growth of tumor sub-regions, beyond use of the RECIST/RANO criteria, and iii) predicting the overall survival from pre-operative mpMRI scans of patients that underwent gross total resection. Finally, we investigate the challenge of identifying the best ML algorithms for each of these tasks, considering that apart from being diverse on each instance of the challenge, the multi-institutional mpMRI BraTS dataset has also been a continuously evolving/growing dataset.

}, url = {https://arxiv.org/pdf/1811.02629.pdf}, author = {Spyridon Bakas and Mauricio Reyes and Andras Jakab and Stefan Bauer and Adri{\`a} Casamitjana and Ver{\'o}nica Vilaplana and et. Al.} } @mastersthesis {xColl-Pol, title = {The Importance of Time in Visual Attention Models}, year = {2018}, abstract = {

Predicting visual attention is a very active field in the computer vision community. Visual attention is a mechanism of the visual system that can select relevant areas within a scene. Models for saliency prediction are intended to automatically predict which regions are likely to be attended by a human observer. Traditionally, ground truth saliency maps are built using only the spatial position of the fixation points, being these xation points the locations where an observer fixates the gaze when viewing a scene. In this work we explore encoding the temporal information as well, and assess it in the application of prediction saliency maps with deep neural networks. It has been observed that the later fi xations in a scanpath are usually selected randomly during visualization, specially in those images with few regions of interest. Therefore, computer vision models have dificulties learning to predict them. In this work, we explore a temporal weighting over the saliency maps to better cope with this random behaviour. The newly proposed saliency representation assigns di erent weights depending on the position in the sequence of gaze fixations, giving more importance to early timesteps than later ones. We used this maps to train MLNet, a state of the art for predicting saliency maps. MLNet predictions were evaluated and compared to the results obtained when the model has been trained using traditional saliency maps. Finally, we show how the temporally weighted saliency maps brought some improvement when used to weight the visual features in an image retrieval task.

The Importance of Time in Visual Attention Models from Universitat Polit{\`e}cnica de Catalunya
}, author = {Coll-Pol, Marta}, editor = {McGuinness, Kevin and Xavier Gir{\'o}-i-Nieto} } @conference {cCamposa, title = {Importance Weighted Evolution Strategies}, booktitle = {NeurIPS 2018 Deep Reinforcement Learning Workshop }, year = {2018}, month = {11/2018}, address = {Montreal, Quebec}, abstract = {

Evolution Strategies (ES) emerged as a scalable alternative to popular Reinforcement Learning (RL) techniques, providing an almost perfect speedup when distributed across hundreds of CPU cores thanks to a reduced communication overhead. Despite providing large improvements in wall-clock time, ES is data inefficient when compared to competing RL methods. One of the main causes of such inefficiency is the collection of large batches of experience, which are discarded after each policy update. In this work, we study how to perform more than one update per batch of experience by means of Importance Sampling while preserving the scalability of the original method. The proposed method, Importance Weighted Evolution Strategies (IW-ES), shows promising results and is a first step towards designing efficient ES algorithms.

}, author = {V{\'\i}ctor Campos and Xavier Gir{\'o}-i-Nieto and Jordi Torres} } @conference {cAlsina, title = {An Interactive Lifelog Search Engine for LSC2018}, booktitle = {Lifelog Search Challenge workshop at ICMR2018}, year = {2018}, month = {06/2018}, publisher = {ACM}, organization = {ACM}, address = {Yokohama, Japan}, abstract = {

In this work, we describe an interactive lifelog search engine developed for the LSC 2018 search challenge at ACM ICMR 2018. The paper introduces the four-step process required to support lifelog search engines and describes the source data for the search engine as well as the approach to ranking chosen for the iterative search engine. Finally the interface used is introduced before we highlight the limits of the current prototype and suggest opportunities for future work.

}, doi = {10.1145/3210539.3210546}, url = {https://dl.acm.org/citation.cfm?id=3210546}, author = {Alsina, Adri{\`a} and Xavier Gir{\'o}-i-Nieto and Gurrin, Cathal} } @mastersthesis {xAlsina, title = {An interactive Lifelog Search Engine for LSC2018}, year = {2018}, abstract = {

This thesis consists on developing an interactive lifelog search engine for the LSC 2018 search\ challenge at ACM ICMR 2018. This search engine is created in order to browse for images from a given lifelog dataset and display them along with some written information related to them and four other images providing contextualization about the searched one.

First of all, the work makes an introduction to the relevance of this project. It introduces the\ reader to the main social problems affronted and the aim of our project to deal with them. Thus, go ahead with the scope of the project introducing to the main objectives fixed. Also, the work is gone by the actual state of the same kind of prototypes that already exist to let the reader see the differences that our project presents.

After the project approach is done, it begins a travel trough the methodology and creation\ process, going deep in the main aspects and the explanation of every election and decision, also remarking the limits of the current prototype.

Additionally, the project concludes with a result section where the system is tested with six users. They are asked to find three specific images using the search engine. This test is divided in two sections: first, a qualitative section where the user is asked to test the system and fill out a survey to see how comfortable it is for him. And a second section, more quantitative, where they value the speed of our system.

Finally, the project concludes going through the actual and future ethics of lifelogging in\ general and with a final conclusion further investigation and future improvement.

}, author = {Alsina, Adri{\`a}}, editor = {Gurrin, Cathal and Xavier Gir{\'o}-i-Nieto} } @article {aDimiccoli18, title = {Introduction to the special issue: Egocentric Vision and Lifelogging}, journal = {Journal of Visual Communication and Image Representation}, year = {2018}, month = {06/2018}, abstract = {

}, doi = {10.1016/j.jvcir.2018.06.010}, url = {https://www.sciencedirect.com/science/article/pii/S1047320318301354}, author = {Dimiccoli, M. and Gurrin, Cathal and Crandall, David and Xavier Gir{\'o}-i-Nieto and Radeva, Petia} } @conference {cRomero-Lopeza, title = {The Impact of Segmentation on the Accuracy and Sensitivity of a Melanoma Classifier based on Skin Lesion Images}, booktitle = {Annual Meeting of the Society of Imaging Informatics in Medicine (SIIM)}, year = {2017}, month = {06/2017}, publisher = {Society of Imaging Informatics for Medicine}, organization = {Society of Imaging Informatics for Medicine}, address = {Pittsburgh, PA, USA}, abstract = {

The accuracy and sensitivity of a Deep Learning based approach for a 2-class classifier for early melanoma detection\ based on skin lesion dermoscopic images increases when\ the classifier is trained with segmented inputs (i.e., images\ containing only the lesions as binary masks, without the\ surrounding context) instead of entire images.

[SIIM 2017 Annual Meeting website]

[SIIM 2017 Session where our work is presented]

The Impact of Segmentation on the Accuracy and Sensitivity of a Melanoma Classifier Based on Skin Lesion Images from Oge Marques
}, url = {http://hdl.handle.net/2117/105582}, author = {Romero-Lopez, Adria and Burdick, Jack and Xavier Gir{\'o}-i-Nieto and Marques, Oge} } @mastersthesis {xArazo, title = {The impact of visual saliency prediction in image classification}, year = {2017}, abstract = {

Advisors: Eva Mohedano, Kevin McGuinness and Xavier Giro-i-Nieto

Program: Master{\textquoteright}s degree in Telecommunications Engineering (MET)

Grade: A (9.7/10.0)

This thesis introduces an architecture to improve the accuracy of a Convolutional Neural Network trained for image classification using visual saliency predictions from the original images. In this thesis the accuracy of a Convolutional Neural Network (CNN) trained for classification has been improved using saliency maps from the original images. The network had an AlexNet architecture and was trained using 1.2 million images from the Imagenet dataset. Two methods had been explored in order to exploit the information from the visual saliency predictions. The first methodologies implemented applied the saliency maps directly to the existing layers of the CNN, which in some cases were already trained for classification and in other they were initialized with random weights. In the second methodology the information from the saliency maps was merged from a new branch, trained at the same time as the initial CNN. In order to speed up the training of the networks the experiments were implemented using images reduced to 128x128. With this sizes the proposed model achieves 12.39\% increase in Top-1 accuracy performance with respect to the original CNN, and additionally reduces the number of parameters needed compared to AlexNet. Regarding the original size images 227x227 a model that increases 1.72\% Top-1 accuracy is proposed. To accelerate the training process of the network the images have been reduced. The methodology that provides the higher improvement in accuracy will be implemented using the original size of the images. The results will be compared to those obtained from the network trained only with the original images. All the methodologies proposed are implemented in a network previously trained for classification. Additionally the most successful methodologies will be implemented in the training of a network. The results will provide information about the best way to add saliency maps to improve the accuracy.

The impact of visual saliency prediction in image classification from Xavier Giro
}, author = {Arazo, Eric}, editor = {McGuinness, Kevin and Mohedano, Eva and Xavier Gir{\'o}-i-Nieto} } @conference {cCasanova16, title = {Interactive Registration Method for 3D data Fusion}, booktitle = {IC3D}, year = {2016}, month = {12/2016}, publisher = {IEEE}, organization = {IEEE}, address = {Li{\`e}ge, Belgium}, abstract = {

Commercial depth sensors represent an opportunity for automation of certain 3D production and analysis tasks. One way to overcome some of their inherent limitations is by capturing the same scene with several depth sensors and merging their data, i.e. by performing 3D data fusion, which requires the registration of point clouds from different sensors. We propose a new interactive, fast and user-friendly method for depth sensor registration. We replace the traditional checkerboard pattern used to extract key points in the scene by a finger detector. This provides a main advantage: the method is easier to use and does not require external objects, while the elapsed time and the registration error are similar to those obtained through the classical method.

We test the proposed approach with an interactive hand tracking application, improved to use more than a single sensor, and we show the increase in detection area by more than 70\%.

}, url = {http://www.3dstereomedia.eu/ic3d}, author = {A. Casanova and A. Pujol-Mir{\'o} and Ruiz-Hidalgo, J. and Casas, J.} } @article {aMohedano, title = {Improving Object Segmentation by using EEG signals and Rapid Serial Visual Presentation}, journal = {Multimedia Tools and Applications}, year = {2015}, month = {07/2015}, abstract = {

This paper explores the potential of brain-computer interfaces in segmenting objects from images. Our approach is centered around designing an effective method for displaying the image parts to the users such that they generate measurable brain reactions. When a block of pixels is displayed, we estimate the probability of that block containing the object of interest using a score based on EEG activity. After several such blocks are displayed in rapid visual serial presentation, the resulting probability map is binarized and combined with the GrabCut algorithm to segment the image into object and background regions. This study extends our previous work that showed how BCI and simple EEG analysis are useful in locating object boundaries in images

}, issn = {1573-7721}, doi = {10.1007/s11042-015-2805-0}, url = {http://dx.doi.org/10.1007/s11042-015-2805-0}, author = {Mohedano, Eva and Healy, Graham and Kevin McGuinness and Xavier Gir{\'o}-i-Nieto and O{\textquoteright}Connor, N. and Smeaton, Alan F.} } @conference {cVentura, title = {Improving Spatial Codification in Semantic Segmentation}, booktitle = {IEEE International Conference on Image Processing (ICIP), 2015}, year = {2015}, month = {09/2015}, publisher = {IEEE}, organization = {IEEE}, address = {Quebec City}, abstract = {

This paper explores novel approaches for improving the spatial codification for the pooling of local descriptors to solve the semantic segmentation problem. We propose to partition the image into three regions for each object to be described: Figure, Border and Ground. This partition aims at minimizing the influence of the image context on the object description and vice versa by introducing an intermediate zone around the object contour. Furthermore, we also propose a richer visual descriptor of the object by applying a Spatial Pyramid over the Figure region. Two novel Spatial Pyramid configurations are explored: Cartesian-based and crown-based Spatial Pyramids. We test these approaches with state-of-the-art techniques and show that they improve the Figure-Ground based pooling in the Pascal VOC 2011 and 2012 semantic segmentation challenges.

Improving Spatial Codification in Semantic Segmentation from Xavier Giro
}, url = {http://arxiv.org/abs/1505.07409}, author = {Ventura, C. and Xavier Gir{\'o}-i-Nieto and Ver{\'o}nica Vilaplana and Kevin McGuinness and Marqu{\'e}s, F. and O{\textquoteright}Connor, N.} } @article {xVentura, title = {Improving Spatial Codification in Semantic Segmentation (Supplementary Material)}, year = {2015}, month = {09/2015}, abstract = {

This document contains supplementary material for the paper "Improving Spatial Codification in Semantic Segmentation" submitted to ICIP 2015. First, there is a section dedicated to the results obtained by categories when ideal object candidates (ground truth masks) are used. Then, an analysis of the results using CPMC and MCG object candidates also detailed by categories. Finally, visual results for CPMC and MCG are showed.

}, author = {Ventura, C. and Xavier Gir{\'o}-i-Nieto and Ver{\'o}nica Vilaplana and Kevin McGuinness and Marqu{\'e}s, F. and Noel E. O{\textquoteright}Connor} } @mastersthesis {xAlfaro15, title = {Inclusion of depth information on a temporal hierarchical co-clustering technique}, year = {2015}, abstract = {

Video segmentation aims to consistently group regions that are similar in appearance and movement along a sequence. This task is an essential step of video analysis and it has important applications in video coding, indexing and retrieval, 3D reconstruction, action recognition, etc. In particular, in this thesis, a multiresolution hierarchical co-clustering technique (MRHC) is analyzed in order to include depth information for improving image segmentation in sequences with small variations. This information is invariant to changes on brightness and/or texture and camera position. Thus, it may correct some errors that are present when segmentation techniques based on color and movement are used, because this information is independent of color information of the image and the movement that occurs in the scene.

Two considerations can be made. Firstly, all the regions of the same object should share similar depth values and, secondly, depth values from regions that do not belong to the same object show discontinuities. Three different ways of coding depth information in MRHC have been studied in this project. In the first approach, the similarity between regions is weighted according to the depth difference between them. The second approach determines the 3D-neighborhood between regions. Finally, a combination of the previous approaches is considered.

The Video Occlusion/Object Boundary Dataset has been used to evaluate the inclusion of the depth on MRHC and to compare this method with the state-of-the-art techniques in the field of video segmentation. The results obtained show that the use of depth information improves the outcome video techniques obtained with the MRHC, outperforming the state-of-the-art methods in this scenario.

}, author = {Alfaro, M.}, editor = {David Varas and Marqu{\'e}s, F.} } @conference {cMcGuinnessa, title = {Insight DCU at TRECVID 2015}, booktitle = {TRECVID 2015 Workshop}, year = {2015}, month = {11/2015}, publisher = {NIST}, organization = {NIST}, address = {Gaithersburg, MD, USA}, abstract = {

Insight-DCU participated in the instance search (INS), semantic indexing (SIN), and localization tasks (LOC) this year.

In the INS task we used deep convolutional network features trained on external data and the query data for this year to train our system. We submitted four runs, three based on convolutional network features, and one based on SIFT/BoW. F A insightdcu 1 was an automatic run using features from the last convolutional layer of a deep network with bag-of-words encoding and achieved 0.123 mAP. F A insightdcu 2 modi ed the previous run to use re-ranking based on an R-CNN model and achieved 0.111 mAP. I A insightdcu 3, our interactive run, achieved 0.269 mAP. Our SIFT-based run F A insightdcu 2 used weak geometric consistency to improve performance over the previous year to 0.187 mAP. Overall we found that using features from the convolutional layers improved performance over features from the fully connected layers used in previous years, and that weak geometric consistency improves performance for local feature ranking.

In the SIN task we again used convolutional network features, this time netuning a network pretrained on external data for the task. We submitted four runs, 2C D A insightdcu.15 1..4 varying the top-level learning algorithm and use of concept co-occurance. 2C D A insightdcu.15 1 used a linear SVM top-level learner, and achieved 0.63 mAP. Exploiting concept co-occurance improved the accuracy of our logistic regression run 2C D A insightdcu.15 3 from 0.058 mAP to 0.6 2C D A insightdcu.15 3.

Our LOC system used training data from IACC.1.B and features similar to our INS run, but using a VLAD encoding instead of a bag-of-words. Unfortunately there was problem with the run that we are still investigating.

Note: UPC and NII participated only in the INS task of this submission.

}, url = {http://www-nlpir.nist.gov/projects/tvpubs/tv.pubs.15.org.html}, author = {Kevin McGuinness and Mohedano, Eva and Amaia Salvador and Zhang, ZhenXing and Marsden, Mark and Wang, Peng and Jargalsaikhan, Iveel and Antony, Joseph and Xavier Gir{\'o}-i-Nieto and Satoh, Shin{\textquoteright}ichi and O{\textquoteright}Connor, N. and Smeaton, Alan F.} } @phdthesis {dPont-Tuset14, title = {Image Segmentation Evaluation and Its Application to Object Detection}, year = {2014}, month = {0/2014}, school = {Universitat Polit{\`e}cnica de Catalunya, BarcelonaTech}, address = {Barcelona}, abstract = {

The first two parts of this Thesis are focused on the study of the supervised evaluation of image segmentation algorithms. Supervised in the sense that the segmentation results are compared to a human-made annotation, known as ground truth, by means of different measures of similarity. The evaluation depends, therefore, on three main points.

First, the image segmentation techniques we evaluate. We review the state of the art in image segmentation, making an explicit difference between those techniques that provide a flat output, that is, a single clustering of the set of pixels into regions; and those that produce a hierarchical segmentation, that is, a tree-like structure that represents regions at different scales from the details to the whole image.

Second, ground-truth databases are of paramount importance in the evaluation. They can be divided into those annotated only at object level, that is, with marked sets of pixels that refer to objects that do not cover the whole image; or those with annotated full partitions, which provide a full clustering of all pixels in an image. Depending on the type of database, we say that the analysis is done from an object perspective or from a partition perspective.

Finally, the similarity measures used to compare the generated results to the ground truth are what will provide us with a quantitative tool to evaluate whether our results are good, and in which way they can be improved. The main contributions of the first parts of the thesis are in the field of the similarity measures.

First of all, from an object perspective, we review the existing measures to compare two object representations and show that some of them are equivalent. In order to evaluate full partitions and hierarchies against an object, one needs to select which of their regions form the object to be assessed. We review and improve these techniques by means of a mathematical model of the problem. This analysis allows us to show that hierarchies can represent objects much better with much less number of regions than flat partitions.

From a partition perspective, the literature about evaluation measures is large and entangled. Our first contribution is to review, structure, and deduplicate the measures available. We pro- vide a new measure that improves previous ones in terms of a set of qualitative and quantitative meta-measures. We also extend the measures on flat partitions to cover hierarchical segmentations.

The third part of this Thesis moves from the evaluation of image segmentation to its application to object detection. In particular, we build on some of the conclusions extracted in the first part to generate segmented object candidates. Given a set of hierarchies, we build the pairs and triplets of regions, we learn to combine the set from each hierarchy, and we rank them using low-level and mid-level cues. We conduct an extensive experimental validation that show that our method outperforms the state of the art in terms of object segmentation quality and object detection accuracy.

}, url = {http://hdl.handle.net/10803/134354}, author = {Jordi Pont-Tuset}, editor = {Marqu{\'e}s, F.} } @conference {cMcGuinness, title = {Insight Centre for Data Analytics (DCU) at TRECVid 2014: Instance Search and Semantic Indexing Tasks}, booktitle = {2014 TRECVID Workshop}, year = {2014}, month = {11/2014}, publisher = {National Institute of Standards and Technology (NIST)}, organization = {National Institute of Standards and Technology (NIST)}, address = {Orlando, Florida (USA)}, abstract = {

Insight-DCU participated in the instance search (INS) and semantic indexing (SIN) tasks in 2014. Two very different approaches were submitted for instance search, one based on features extracted using pre-trained deep convolutional neural networks (CNNs), and another based on local SIFT features, large vocabulary visual bag-of-words aggregation, inverted index-based lookup, and geometric verification on the top-N retrieved results. Two interactive runs and two automatic runs were submitted, the best interactive runs achieved a mAP of 0.135 and the best automatic 0.12. Our semantic indexing runs were based also on using convolutional neural network features, and on Support Vector Machine classifiers with linear and RBF kernels. One run was submitted to the main task, two to the no annotation task, and one to the progress task. Data for the no-annotation task was gathered from Google Images and ImageNet. The main task run has achieved a mAP of 0.086, the best no-annotation runs had a close performance to the main run by achieving a mAP of 0.080, while the progress run had 0.043.

[2014 TREC Video Retrieval Evaluation Notebook Papers and Slides]

Poster

}, url = {http://hdl.handle.net/2117/24915}, author = {Kevin McGuinness and Mohedano, Eva and Zhang, ZhenXing and Hu, Feiyan and Albatal, Rami and Gurrin, Cathal and O{\textquoteright}Connor, N. and Smeaton, Alan F. and Amaia Salvador and Xavier Gir{\'o}-i-Nieto and Ventura, C.} } @mastersthesis {xPereira, title = {An investigation of eye gaze tracking utilities in image object recognition}, year = {2014}, abstract = {

Computer vision has been one of the most revolutionary technologies of the last few decades. This project investigates how to improve an image recognition system (image classifier) using a not very exploded technology; eye gaze tracking. The aim of this project is to explore the benefits that this technology can bring to an image classifier. The experiment that is set in this project is to build a dataset with an eye tracking device and (using different sized cropped parts of the image based on the eye tracking data) see how the performance of an image classifier is affected with these images. The results are interesting. Since smaller images have to be processed by using this method, the system is more efficient. Regarding the performance, it is very similar to the one obtained without using any eye tracking data, so it is arguable to state that it presents an improvement, and opens new directions of investigation for future works.

Poster


}, author = {Imedio-Pereira, Sergi}, editor = {Mohedano, Eva and Xavier Gir{\'o}-i-Nieto and Albatal, Rami and O{\textquoteright}Connor, N.} } @article {aVentura13, title = {Improving retrieval accuracy of Hierarchical Cellular Trees for generic metric spaces}, journal = {Multimedia Tools and Applications}, year = {2013}, abstract = {

Metric Access Methods (MAMs) are indexing techniques which allow working in generic metric spaces. Therefore, MAMs are specially useful for Content-Based Image Retrieval systems based on features which use non Lp norms as similarity measures. MAMs naturally allow the design of image browsers due to their inherent hierarchical structure. The Hierarchical Cellular Tree (HCT), a MAM based indexing technique, provides the starting point of our work. In this paper, we describe some limitations detected in the original formulation of the HCT and propose some modifications to both the index building and the search algorithm. First, the covering radius, which is defined as the distance from the representative to the furthest element in a node,may not cover all the elements belonging to the node{\textquoteright}s subtree. Therefore, we propose to redefine the covering radius as the distance from the representative to the furthest element in the node{\textquoteright}s subtree. This new definition is essential to guarantee a correct construction of the HCT. Second, the proposed Progressive Query retrieval scheme can be redesigned to perform the nearest neighbor operation in a more efficient way. We propose a new retrieval scheme which takes advantage of the benefits of the search algorithm used in the index building. Furthermore, while the evaluation of the HCT in the original work was only subjective, we propose an objective evaluation based on two aspects which are crucial in any approximate search algorithm: the retrieval time and the retrieval accuracy. Finally, we illustrate the usefulness of the proposal by presenting some actual applications.

}, doi = {10.1007/s11042-013-1686-3}, author = {Ventura, C. and Ver{\'o}nica Vilaplana and Xavier Gir{\'o}-i-Nieto and Marqu{\'e}s, F.} } @mastersthesis {xMohedano13, title = {Investigating EEG for Saliency and Segmentation Applications in Image Processing}, year = {2013}, abstract = {

Advisors: Kevin McGuinness, Xavier Gir{\'o}-i-Nieto, Noel O{\textquoteright}Connor

School: Dublin City University (Ireland)

The main objective of this project is to implement a new way to compute saliency maps and to locate an object in an image by using a brain-computer interface. To achieve this, the project is centered in designing the proper way to display the different parts of the images to the users in such a way that they generate measurable reactions. Once an image window is shown, the objective is to compute a score based on the EEG activity and compare its result with the current automatic methods to estimate saliency maps. Also, the aim of this work is to use the EEG map as a seed for another segmentation algorithm that will extract the object from the background in an image. This study provides evidence that BCI are useful to find the location of the objects in a simple images via straightforward EEG analysis and this represents the starting point to locate objects in more complex images.

Related post on BitSearch.

Eva Mohedano, "Investigating EEG for Saliency and Segmentation Applications in Image Processing" from Xavi Gir{\'o}
}, keywords = {Brain-computer interfaces (BCI), Eelectroencephalography (EEG), rapid serial visual presentation (RSVP), saliency map, segmentation}, author = {Mohedano, Eva}, editor = {Kevin McGuinness and Xavier Gir{\'o}-i-Nieto and O{\textquoteright}Connor, N.} } @inbook {bCalderero12 , title = {Image Analysis and Understanding Based on Information Theoretical Region Merging Approaches for Segmentation and Cooperative Fusion}, booktitle = {Handbook of Research on Computational Intelligence for Engineering, Science, and Business}, year = {2012}, pages = {75-121}, publisher = {IGI Global}, organization = {IGI Global}, chapter = {4}, abstract = {

This chapter addresses the automatic creation of simplified versions of the image, known as image segmentation or partition, preserving the most semantically relevant information of the image at different levels of analysis. From a semantic and practical perspective, image segmentation is a first and key step for image analysis and pattern recognition since region-based image representations provide a first level of abstraction and a reduction of the number of primitives, leading to a more robust estimation of parameters and descriptors. The proposed solution is based on an important class of hierarchical bottom-up segmentation approaches, known as region merging techniques. These approaches naturally provide a bottom-up hierarchy, more suitable when no a priori information about the image is available, and an excellent compromise between efficiency of computation and representation. The chapter is organized in two parts dealing with the following objectives: (i) provide an unsupervised solution to the segmentation of generic images; (ii) design a generic and scalable scheme to automatically fuse hierarchical segmentation results that increases the robustness and accuracy of the final solution

}, isbn = {978-1-466-62518-1}, doi = {10.4018/978-1-4666-2518-1.ch004}, author = {Calderero, F. and Marqu{\'e}s, F.} } @article {bSuau12, title = {INTAIRACT: Joint Hand Gesture and Fingertip Classification for Touchless Interaction}, journal = {Lecture notes in computer science}, volume = {7585}, year = {2012}, month = {10/2012}, pages = {602-606}, publisher = {Springer}, chapter = {3}, address = {Heidelberg}, abstract = {

In this demo we present INTAIRACT, an online hand-based touchless interaction system. Interactions are based on easy-to-learn hand gestures, that combined with translations and rotations render a user friendly and highly configurable system. The main advantage with respect to existing approaches is that we are able to robustly locate and identify fingertips. Hence, we are able to employ a simple but powerful alphabet of gestures not only by determining the number of visible fingers in a gesture, but also which fingers are being observed. To achieve such a system we propose a novel method that jointly infers hand gestures and fingertip locations using a single depth image from a consumer depth camera. Our approach is based on a novel descriptor for depth data, the Oriented Radial Distribution (ORD) [1]. On the one hand, we exploit the ORD for robust classification of hand gestures by means of efficient k-NN retrieval. On the other hand, maxima of the ORD are used to perform structured inference of fingertip locations. The proposed method outperforms other state-of-the-art approaches both in gesture recognition and fingertip localization. An implementation of the ORD extraction on a GPU yields a real-time demo running at approximately 17fps on a single laptop.

1. Suau, X., Ruiz-Hidalgo, J., Casas, J.R.: Oriented Radial Distribution on Depth\ Data: Application to the Detection of End-Effectors. In: ICASSP. (2012)

}, isbn = {978-3-642-33885-4}, issn = {978-3-642-33884-7}, doi = {10.1007/978-3-642-33885-4_62}, author = {Suau, X. and Alcoverro, M. and L{\'o}pez-M{\'e}ndez, A. and Ruiz-Hidalgo, J. and Casas, J.} } @conference {cGiro-i-Nieto12a, title = {Interactive segmentation and tracking of video objects}, booktitle = {Image Analysis for Multimedia Interactive Services (WIAMIS), 2012 13th International Workshop on}, year = {2012}, month = {05/2012}, publisher = {IEEE}, organization = {IEEE}, address = {Dublin, Ireland}, abstract = {

This paper describes a mechanism to interactively segment objects from a sequence of video frames. The extracted object can be later embedded in a different background, associated to local scale metadata or used to train an automatic object detector. The workflow requires the interaction of the user at two stages: the temporal segmentation of the frames contain- ing the object and the generation of an object mask to initial- ize a video tracker. The mask is defined as a combination of regions generated by an image segmentation algorithm. This framework has been integrated in an annotation tool available to the public.\ 

}, keywords = {Image segmentation, Object segmentation, Proposals, semantics, Signal processing algorithms, Video sequences, Visualization}, doi = {10.1109/WIAMIS.2012.6226749}, url = {http://dx.doi.org/10.1109/WIAMIS.2012.6226749}, author = {Xavier Gir{\'o}-i-Nieto and Martos, Manel} } @conference {cBragos11a, title = {Implementation and first results of the Introduction to Engineering course in the ETSETB-UPC new degrees}, booktitle = {II Conferencia Internacional en Fomento e Innovaci{\'o}n con Nuevas Tecnolog{\'\i}as en la Docencia de la Ingenier{\'\i}a}, year = {2011}, pages = {1{\textendash}4}, isbn = {978-1-4577-0559-5}, doi = {10.1109/FINTDI.2011.5945971}, url = {http://ieeexplore.ieee.org/xpls/abs_all.jsp?arnumber=5945971}, author = {Bragos, R. and Pegueroles, J. and Alarc{\'o}n, E. and Camps, A. and Sarda, J. and Consolacion, C. and Mussons, J. and Pons, O. and Albert Oliveras and Garc{\'\i}a, M. and Onrubia, R. and Elisa Sayrol} } @conference {cValero11, title = {Improved binary partition tree construction for hyperspectral images: application to object detection}, booktitle = {IEEE International Geoscience and Remote Sensing Symposium, IGARSS 2011}, year = {2011}, pages = {2515{\textendash}2518}, address = {Vancouver, Canada}, doi = {10.1109/IGARSS.2011.6049723}, url = {http://ieeexplore.ieee.org/xpl/mostRecentIssue.jsp?punumber=6034618}, author = {Valero, S. and Salembier, P. and Chanussot, J. and Cuadres, C.} } @mastersthesis {xTella11, title = {Interactive Image Processing demonstrations for the web}, year = {2011}, abstract = {

This diploma thesis aims to provide a framework for developing web applications for ImagePlus, the software develpment platform in C++ of the Image Processing Group of the Technical University of Catalonia (UPC). These web applications are to demonstrate the functionality of the image processing algorithms to any visitor to the group website. Developers are also benefited from this graphical user interface because they can easily create Graphical User Interfaces (GUIs) for the processing algorithms

Interactive Image Processing Demos for the Web from Xavi Gir{\'o}
}, keywords = {c++, demo, image processing, web}, url = {http://hdl.handle.net/2099.1/13540}, author = {Tella, Marcel}, editor = {Xavier Gir{\'o}-i-Nieto} } @mastersthesis {x11a, title = {Interactive Image Processing Demos for the Web}, year = {2011}, abstract = {

This diploma thesis aims to provide a framework for developing web applications for ImagePlus, the software develpment platform in C++ of the Image Processing Group of the Technical University of Catalonia (UPC). These web applications are to demonstrate the functionality of the image processing algorithms to any visitor to the group website. Developers are also benefited from this graphical user interface because they can easily create Graphical User Interfaces (GUIs) for the processing algorithms.



}, url = {http://upcommons.upc.edu/pfc/handle/2099.1/13540} } @mastersthesis {xVentura10, title = {Image-Based Query by Example Using MPEG-7 Visual Descriptors}, year = {2010}, abstract = {

This project presents the design and implementation of a Content-Based Image Re- trieval (CBIR) system where queries are formulated by visual examples through a graphical interface. Visual descriptors and similarity measures implemented in this work followed mainly those defined in the MPEG-7 standard although, when necessary, extensions are proposed. Despite the fact that this is an image-based system, all the proposed descriptors have been implemented for both image and region queries, allowing the future system upgrade to support region-based queries. This way, even a contour shape descriptor has been developed, which has no sense for the whole image. The system has been assessed on different benchmark databases; namely, MPEG-7 Common Color Dataset, and Corel Dataset. The evaluation has been performed for isolated descriptors as well as for combinations of them. The strategy studied in this work to gather the information obtained from the whole set of computed descriptors is weighting the rank list for each isolated descriptor.\ 

}, url = {http://upcommons.upc.edu/pfc/handle/2099.1/9453}, author = {Ventura, C.}, editor = {Marqu{\'e}s, F. and Jordi Pont-Tuset} } @article {aGudmundsson10, title = {Improved 3D reconstruction in smart-room environments using ToF imaging}, journal = {Computer vision and image understanding}, volume = {114}, number = {12}, year = {2010}, month = {12/2010}, pages = {1376{\textendash}1384}, abstract = {

This paper presents a general analysis framework towards exploiting the underlying hierarchical and scalable structure of an articulated object for pose estimation and tracking. Scalable human body models are introduced as an ordered set of articulated models fulfilling an inclusive hierarchy. The concept of annealing is applied to derive a generic particle filtering scheme able to perform a sequential filtering over the set of models contained in the scalable human body model. Two annealing loops are employed, the standard likelihood annealing and the newly introduced structural annealing, leading to a robust, progressive and efficient analysis of the input data. The validity of this scheme is tested by performing markerless human motion capture in a multi-camera environment employing the standard HumanEva annotated datasets. Finally, quantitative results are presented and compared with other existing HMC techniques.

}, issn = {1077-3142}, doi = {10.1016/j.cviu.2010.07.011}, url = {http://www.sciencedirect.com/science/article/pii/S1077314210001736}, author = {Gudmundsson, S. and M. Pard{\`a}s and Casas, J. and Sveinsson, J. and Aanaes, H. and Larsen, R.} } @conference {cRolon10, title = {Improved local pdf estimation in the wavelet domain for generalized lifting}, booktitle = {Picture Coding Symposium, PCS 2010}, year = {2010}, address = {Nagoya, Japan}, isbn = {0-7803-3192-3}, url = {http://hdl.handle.net/2117/11675}, author = {Rol{\'o}n, J. and Salembier, P.} } @phdthesis {dCalderero10, title = {Information Theoretical Region Merging Approaches and Fusion of Hierarchical Image Segmentation Results}, year = {2010}, school = {Universitat Polit{\`e}cnica de Catalunya (UPC)}, type = {phd}, abstract = {This PhD. thesis addresses the unsupervised hierarchical segmentation of images and the automatic extraction of the image partitions providing the most semantically relevant explanations of the image at different levels of analysis. From a semantic and practical perspective, image segmentation is a first and key step for image analysis and pattern recognition since region-based image representations provide a first level of abstraction and a reduction of the number of primitives, leading to a more robust estimation of parameters and descriptors. The proposal of this dissertation is based on an important class of hierarchical bottomup segmentation approaches, known as region merging techniques. These approaches naturally provide a bottom-up hierarchy, more suitable when no a priori information about the image is available, and an excellent compromise between efficiency of computation and representation. The first part of the dissertation is devoted to the proposal, under a statistical framework, of a family of unsupervised region merging techniques. These techniques are characterized by general and non-parametric region models, with neither color nor texture homogeneity assumptions, and a set of innovative merging criteria, based on information theory statistical measures. The scale consistency of the partitions is assured through (i) a size regularization term into the merging criteria and a classical merging order, or (ii) using a novel scale-based merging order to avoid the region size homogeneity imposed by the use of a size regularization term. Moreover, a partition significance index is defined to automatically determine the subset of most representative partitions from the created hierarchy. Most significant automatically extracted partitions show the ability to represent the semantic content of the image. Results are promising, outperforming in most indicators both object-oriented and texture state-of-the-art segmentation techniques. The second part of the thesis is focused on the fusion of hierarchical segmentation results, obtained by different segmentation techniques or from different information channels of the same image, with the purpose of increasing the robustness and accuracy of the final solution. In this case, a generic and scalable segmentation scheme based on a cooperative principle, named cooperative region merging, is designed to combine in an unsupervised manner a set of hierarchical region-based representations. The intuition behind this cooperative approach is to iteratively establish a basic or conservative consensus between the independent techniques that can be used as the starting point from which further consensus may be built, similar to a negotiation process for decision making. In addition to the new fused hierarchy of partitions, the proposed scheme automatically provides a subset of partitions considered most relevant from the fusion viewpoint. The combination of hierarchical segmentation results applying different segmentation techniques over the same color image leads to a global improvement of the accuracy and the stability of the segmentation results. Moreover, the use of the cooperative approach for the fusion of segmentation results from heterogenous information channels is presented. Application examples demonstrate the high flexibility and potentiality of the cooperative region merging scheme into a wide range of applications and fusion problems (for instance, in multiview processing and remote sensing). The proposed fusion strategies are able to naturally incorporate a priori available knowledge on the types of information to combine, or on the specificities of the particular fusion problem, and to improve the efficiency and reduce the computational load of the fusion process without compromising the accuracy of the segmentation results.}, author = {Calderero, F.}, editor = {Marqu{\'e}s, F.} } @mastersthesis {x10, title = {Interf{\'\i}cie gr{\`a}fica d{\textquoteright}usuari per a l{\textquoteright}avaluaci{\'o} de classificadors d{\textquoteright}imatges}, year = {2010}, abstract = {

Aquest projecte final de carrera s{\textquoteright}ha realitzat amb l{\textquoteright}objectiu de crear una interf{\'\i}cie gr{\`a}fica capa{\c c} d{\textquoteright}avaluar diferents classificadors d{\textquoteright}una forma gr{\`a}fica i molt intu{\"\i}tiva. La interf{\'\i}cie gr{\`a}fica s{\textquoteright}ha d{\textquoteright}implementar dins del Graphic Annotation Tool (GAT), una aplicaci{\'o} de programari lliure creada pel Grup de Processament de la Imatge de la UPC. El projecte es basa en la creaci{\'o} d{\textquoteright}unes anotacions manuals per tal de realitzar un entrenament a partir d{\textquoteright}un algoritme d{\textquoteright}aprenentatge supervisat. Els resultats d{\textquoteright}aquesta classificaci{\'o} seran visualitzats per la nova pestanya del GAT anomenada Classifier. Aix{\`o} ser{\`a} possible gr{\`a}cies a la integraci{\'o} dels motors de d{\textquoteright}entrenament i de detecci{\'o}, a la incorporaci{\'o} del algoritme de validaci{\'o} creuada aix{\'\i} com que sigui possible triar entre diferents classificadors. Per tal d{\textquoteright}avaluar aquests resultats es calculen la precisi{\'o}, el record, els falsos positius i els falsos negatius.

}, keywords = {image classification, pattern recognition}, url = {http://hdl.handle.net/2099.1/13526}, author = {Gimeno, Mireia}, editor = {Xavier Gir{\'o}-i-Nieto} } @inbook {bCanton-Ferrer09, title = {Image and video processing tools for HCI}, booktitle = {Multimodal signal processing: theory and applications for human-computer interaction}, year = {2009}, pages = {93{\textendash}118}, isbn = {9780123748256}, author = {Cristian Canton-Ferrer and M. Pard{\`a}s and Ver{\'o}nica Vilaplana} } @conference {cButko09, title = {Improving Detection of Acoustic Events Using Audiovisual Data and Feature Level Fusion}, booktitle = {10th Annual Conference of the International Speech Communication Association}, year = {2009}, pages = {1147{\textendash}1150}, isbn = {978-1-61567-692-7}, url = {http://gps-tsc.upc.es/imatge/_Xgiro/research/publications/2009/interspeech.pdf}, author = {Butko, T. and Cristian Canton-Ferrer and Segura, C. and Xavier Gir{\'o}-i-Nieto and Nadeu, C. and Hernando, J. and Casas, J.} } @conference {cSanchez-Riera09, title = {Indoor PTZ camera calibration with concurrent PT axes}, booktitle = {Fourth International Conference on Computer Vision Theory and Applications}, year = {2009}, pages = {45{\textendash}50}, abstract = {

The introduction of active (pan-tilt-zoom or PTZ) cameras in Smart Rooms in addition to fixed static cameras allows to improve resolution in volumetric reconstruction, adding the capability to track smaller objects with higher precision in actual 3D world coordinates. To accomplish this goal, precise camera calibration data should be available for any pan, tilt, and zoom settings of each PTZ camera. The PTZ calibration method proposed in this paper introduces a novel solution to the problem of computing extrinsic and intrinsic parameters for active cameras. We first determine the rotation center of the camera expressed under an arbitrary world coordinate origin. Then, we obtain an equation relating any rotation of the camera with the movement of the principal point to define extrinsic parameters for any value of pan and tilt. Once this position is determined, we compute how intrinsic parameters change as a function of zoom. We validate our method by evaluating the re-projection error and its stability for points inside and outside the calibration set.

}, isbn = {0-8186-7310-9}, doi = {10.5220/0001754900100015}, url = {http://visapp.visigrapp.org/Abstracts/2009/VISAPP_2009_Abstracts.htm}, author = {Sanchez-Riera, J. and Salvador, J. and Casas, J.} } @article {aNeumann09, title = {Integration of audiovisual sensors and technologies in a smart room}, journal = {Personal and ubiquitous computing}, volume = {13}, number = {1}, year = {2009}, pages = {15{\textendash}23}, abstract = {

At the Technical University of Catalonia\ (UPC), a smart room has been equipped with 85 microphones and 8 cameras. This paper describes the setup of the\ sensors, gives an overview of the underlying hardware and\ software infrastructure and indicates possibilities for highand low-level multi-modal interaction. An example of\ usage of the information collected from the distributed\ sensor network is explained in detail: the system supports\ a group of students that have to solve a lab assignment\ related problem.

}, keywords = {CHIL, PROVEC}, issn = {1617-4909}, doi = {10.1007/s00779-007-0172-1}, url = {http://hdl.handle.net/2117/9468}, author = {Neumann, J. and Casas, J. and Macho, D. and Ruiz-Hidalgo, J.} } @mastersthesis {dCortes09, title = {Interfaz gr{\'a}fica de usuario para la b{\'u}squeda de im{\'a}genes basada en im{\'a}genes}, year = {2009}, month = {07/2009}, type = {BSc}, abstract = {

El Proyecto de Final de Carrera (PFC) responde a esa necesidad de creaci{\'o}n de herramientas de acceso a contenido multimedia, nuevas herramientas que faciliten la recuperaci{\'o}n de toda esa informaci{\'o}n audiovisual almacenada. El Graphic Object Searcher (GOS) es una interfaz gr{\'a}fica para realizar b{\'u}squedas de im{\'a}genes alojadas en grandes bases de datos a partir de una imagen ejemplo y de unos criterios de b{\'u}squeda establecidos por el usuario. La realizaci{\'o}n de PFC permite trabajar en las dos {\'a}reas tecnol{\'o}gicas con m{\'a}s auge actualmente: el sector multimedia (gesti{\'o}n de contenido audiovisual) y las tecnolog{\'\i}as de la informaci{\'o}n (TIC) (inform{\'a}tica al servicio de la comunicaci{\'o}n). Estas dos {\'a}reas tienden a aunar esfuerzos en una sociedad abocada al uso y consumo de contenido audiovisual a trav{\'e}s de m{\'u}ltiples plataformas y dispositivos en cualquier sector econ{\'o}mico y social (ocio, formaci{\'o}n, servicios, etc.). Cualquier profesional del sector audiovisual ha de adquirir conocimiento y experiencia en ambas {\'a}reas para cimentar su carrera.

Interfaz gr{\'a}fica de usuario para la b{\'u}squeda de im{\'a}genes basada en im{\'a}genes from Xavi Gir{\'o}
}, url = {http://hdl.handle.net/2099.1/8588}, author = {Cort{\'e}s, S.}, editor = {Xavier Gir{\'o}-i-Nieto} } @conference {cRolon08, title = {Image compression with generalized lifting and partial knowledge of the signal pdf}, booktitle = {IEEE International Conference on Image Processing, ICIP 2008}, year = {2008}, pages = {250{\textendash}254}, address = {San Diego, USA}, isbn = {88-86179-83-9}, author = {Rol{\'o}n, J. and Salembier, P. and Alameda, X.} } @conference {cButko08a, title = {Inclusion of video information for detection of acoustic events using the fuzzy integral}, booktitle = {Machine Learning for Multimodal Interaction: 5th International Workshop}, year = {2008}, pages = {74{\textendash}85}, isbn = {978-3540858522}, author = {Butko, T. and Temko, A. and Nadeu, C. and Cristian Canton-Ferrer} } @conference {cSalvador06, title = {Image-adapted voxelization in multicamera settings}, booktitle = {International Workshop on Multimedia Signal Processing}, year = {2006}, pages = {161{\textendash}165}, abstract = {

This paper presents a new procedure to explore 3D for visual analysis in multicamera environments. We propose a 3D geometry to sample (voxelize) the space when checking the consistency of the analyzed features in the multiple views. Contrary to regular voxelization, the proposed geometry is irregular in 3D, but becomes regular once projected onto camera images. The aim is to better exploit the redundancy and the enriched information provided in multiview frameworks at the analysis stage. This is accomplished by balancing the usage of the available data (i.e. captured pixels) across the multiple cameras, instead of focusing in a regular sampling of the 3D space from which we do not have direct data. An efficient recursive algorithm that uses the proposed geometry is outlined, and the experimental results reflect higher accuracy than regular voxelization with equivalent restrictions for the chosen multiview analysis applications

}, isbn = {0-7803-3258-X}, author = {Salvador, J. and Casas, J.} } @conference {cCasas06, title = {Image-based Multi-view Scene Analysis using {\textquoteright}Conexels{\textquoteright}}, booktitle = {HCSNet Workshop on the Use of Vision in Human-Computer Interaction (VisHCI 2006)}, year = {2006}, pages = {203{\textendash}212}, abstract = {

Multi-camera environments allow constructing volumetric models of the scene to improve the analysis performance of computer vision algorithms (e.g. disambiguating occlusion). When representing volumetric results of image-based multi-camera analysis, a direct approach is to scan the 3D space with regular voxels. Regular voxelization is good at high spatial resolutions for applications such as volume visualization and rendering of synthetic scenes generated by geometric models, or to represent data resulting from direct 3D data capture (e.g. MRI). However, regular voxelization shows a number of drawbacks for visual scene analysis, where direct measurements on 3D voxels are not usually available. In this case, voxel values are computed rather as a result of the analysis on ?projected? image data.

In this paper, we first provide some statistics to show how voxels project to ?unbalanced? sets of image data in common multi-view analysis settings. Then, we propose a 3D geometry for multi-view scene analysis providing a better balance in terms of the number of pixels used to analyse each elementary volumetric unit. The proposed geometry is non-regular in 3D space, but becomes regular once projected onto camera images, adapting the sampling to the images. The aim is to better exploit multiview image data by balancing its usage across multiple cameras instead of focusing in regular sampling of 3D space, from which we do not have direct measurements. An efficient recursive algorithm using the proposed geometry is outlined. Experimental results reflect better balance and higher accuracy for multi-view analysis than regular voxelization with equivalent restrictions.

}, keywords = {epipolar geometry, Multi-view analysis, volume voxelization}, isbn = {1-920-68238-4}, url = {http://crpit.com/Vol56.html}, author = {Casas, J. and Salvador, J.} } @inbook {bPardas04, title = {The InterFace Software Platform for Interactive Virtual Characters}, booktitle = {Mpeg-4 facial animation: the standard, implementation and applications}, year = {2004}, pages = {169{\textendash}183}, isbn = {0-470-84465-5}, author = {M. Pard{\`a}s and Pandzic, I. and Cannella, M. and Davoine, F. and Forchheimer, R. and Lavagetto, F. and Marriott, A. and Malassiotis, S.} } @book {eSalembier02, title = {Introduction to the mpeg-7: multimedia content description interface}, year = {2002}, publisher = {Wiley}, organization = {Wiley}, isbn = {0471486787}, author = {Manjunath, B.S and Salembier, P. and Sikora, T} } @book {eMarques00, title = {Introduction fo interpretation of graphic images}, year = {2000}, isbn = {0-8194-2380-7}, editor = {Marqu{\'e}s, F.} } @article {pMarques97, title = {Image Segmentation and Object Tracking Method and Corresponding System}, number = {974025587}, year = {1997}, month = {10/1997}, type = {Invention patent}, address = {International}, issn = {974025587}, author = {Marqu{\'e}s, F. and Molina, C.} } @conference {cSalembier97, title = {Image sequence analysis and merging algorithms}, booktitle = {International Workshop on Very Low Bit-rate Video, VLBV{\textquoteright}97}, year = {1997}, pages = {1{\textendash}8}, address = {Link{\"o}ping, Sweden}, isbn = {0-7803-9752-5}, author = {Salembier, P. and Garrido, L. and Garc{\'\i}a, D.} } @phdthesis {dCasas96, title = {Image compression based on perceptual coding techniques}, volume = {PhD}, year = {1996}, month = {03/1996}, pages = {178}, school = {Universitat Polit{\`e}cnica de Catalunya (UPC)}, type = {phd}, address = {download link}, abstract = {

This thesis studies image and video sequence coding methods from the point of view of the way the human visual system perceives and understands visual information. The relevance of such study is due, on the one hand, to the important role that visual signals have in our civilization and, on the other hand, to the problem of representing the large amount of data that image and video processing systems have to deal with. Three different approaches have been investigated for the coding of image textures in an advanced compression scheme relying in aspects of visual perception. The first approach is based on image transitions and the interpolation of smooth areas from such transitions. The second one, considers the extraction, selection and coding of meaningful image details. Finally, the third approach studies the efficient representation of homogeneous fine textures that give a natural appearance to the reconstructed images at high compression levels. In order to apply these techniques for still image and video coding, a three component model of the image, that matches the perceptual properties of the human vision, is put forward. The coding approaches subject of research have leaded to the design of three new image analysis and coding techniques. Using non-linear tools from the framework of Mathematical Morphology, three texture coding techniques are developed. In particular, - A "morphological" image interpolation method aimed at the problem of scattered data interpolation. - An empirical subjective criterion for the ranking and selection of image details according to visual perception. - The application of a conventional image coding technique, subband coding, to the coding of arbitrarily shaped image regions (region-based subband coding). These are new texture coding techniques in the field of object-oriented and Second Generation image and video coding schemes. Furthermore, the model of the image that has been investigated follows the line of the last proposals in the framework of MPEG4, the forthcoming coding standard for low bit-rate visual communications, which considers the possibility of content-based manipulation and coding of visual information.

The PhD thesis report can be downloaded from http://hdl.handle.net/10803/6920

}, url = {http://hdl.handle.net/10803/6920}, author = {Casas, J.}, editor = {Torres, L.} } @article {aSayrol95, title = {Image Restoration using the W-Slice Method}, journal = {IEEE transactions on image processing}, volume = {4}, number = {4}, year = {1995}, pages = {1174{\textendash}1181}, abstract = {

We propose the use of higher order statistics (HOS)-based methods to address the problem of image restoration. The restoration strategy is based on the fact that the phase information of the original image and its HOS are not distorted by some types of blurring. The difficulties associated with the combination of 2-D signals and their HOS are reduced by means of the Radon transform. Two methods that apply the weight-slice algorithm over the projections are developed. Simulation results illustrate the performance of the proposed methods

}, issn = {1057-7149}, author = {Elisa Sayrol and Gasull, A. and Nikias, C.} } @conference {cMarques95a, title = {Interpolation and extrapolation of iomage partitions using fourier descriptions: to segmentation-based coding schemes}, booktitle = {IEEE INTERNATIONAL CONFERENCE ON IMAGE PROCESSING{\textquoteright}95.}, year = {1995}, pages = {584{\textendash}587}, author = {Marqu{\'e}s, F. and Llorens, B. and Gasull, A.} } @conference {cMarques94c, title = {Interleaved segmentation and motion estimation by means of morphological tools}, booktitle = {Workshop on Image Analysis and Synthesis in Image Coding}, year = {1994}, address = {Berlin, Germany}, author = {Marqu{\'e}s, F. and Bouchard, L. and Corset, I. and Jeannin, S. and Morros, J.R. and M. Pard{\`a}s and Salembier, P. and Torres, L.} } @conference {cSayrol93a, title = {Image analysis using higher-order statistics and the radon transfrom}, booktitle = {IEEE SIGNAL PROCESSING WORKSHOP ON HIGH-ORDER-STATISTICS}, year = {1993}, pages = {76{\textendash}80}, author = {Elisa Sayrol and Nikias, C. and Gasull, A.} } @conference {cSayrol93, title = {Image analysis using higher-order statistics and the random transfor}, booktitle = {WORKSHOP ON HIGHER-ORDER STATISTICS}, year = {1993}, author = {Elisa Sayrol and Gasull, A.} }