@article {aBellver, title = {Mask-guided sample selection for Semi-Supervised Instance Segmentation}, journal = {Multimedia Tools and Applications}, year = {2020}, month = {07/2020}, abstract = {

Image segmentation methods are usually trained with pixel-level annotations, which require significant human effort to collect. The most common solution to address this constraint is to implement weakly-supervised pipelines trained with lower forms of supervision, such as bounding boxes or scribbles. Another option are semi-supervised methods, which leverage a large amount of unlabeled data and a limited number of strongly-labeled samples. In this second setup, samples to be strongly-annotated can be selected randomly or with an active learning mechanism that chooses the ones that will maximize the model performance.\ In this work, we propose a sample selection approach to decide which samples to annotate for semi-supervised instance segmentation. Our method consists in first predicting pseudo-masks for the unlabeled pool of samples, together with a score predicting the quality of the mask. This score is an estimate of the Intersection Over Union (IoU) of the segment with the ground truth mask. We study which samples are better to annotate given the quality score, and show how our approach outperforms a random selection, leading to improved performance for semi-supervised instance segmentation with low annotation budgets.

}, doi = {10.1007/s11042-020-09235-4}, url = {http://link.springer.com/article/10.1007/s11042-020-09235-4}, author = {M{\'\i}riam Bellver and Amaia Salvador and Jordi Torres and Xavier Gir{\'o}-i-Nieto} } @conference {cBellverb, title = {Budget-aware Semi-Supervised Semantic and Instance Segmentation}, booktitle = {CVPR 2019 DeepVision Workshop}, year = {2019}, month = {05/2019}, publisher = {OpenCVF}, organization = {OpenCVF}, address = {Long Beach, CA, USA}, abstract = {

Methods that move towards less supervised scenarios are key for image segmentation, as dense labels demand significant human intervention. Generally, the annotation burden is mitigated by labeling datasets with weaker forms of supervision, e.g. image-level labels or bounding boxes. Another option are semi-supervised settings, that commonly leverage a few strong annotations and a huge number of unlabeled/weakly-labeled data. In this paper, we revisit semi-supervised segmentation schemes and narrow down significantly the annotation budget (in terms of total labeling time of the training set) compared to previous approaches. With a very simple pipeline, we demonstrate that at low annotation budgets, semi-supervised methods outperform by a wide margin weakly-supervised ones for both semantic and instance segmentation. Our approach also outperforms previous semi-supervised works at a much reduced labeling cost. We present results for the Pascal VOC benchmark and unify weakly and semi-supervised approaches by considering the total annotation budget, thus allowing a fairer comparison between methods.

Winners of the best paper award at the $\#$CVPR2019 DeepVision workshop

Budget-aware Semi-Supervised Semantic and Instance Segmentation from Universitat Polit{\`e}cnica de Catalunya
}, url = {https://arxiv.org/abs/1905.05880}, author = {M{\'\i}riam Bellver and Amaia Salvador and Jordi Torres and Xavier Gir{\'o}-i-Nieto} } @phdthesis {dSalvador, title = {Computer Vision beyond the visible: Image understanding through language}, volume = {Phd}, year = {2019}, school = {Universitat Politecnica de Catalunya}, address = {Barcelona}, abstract = {

In the past decade, deep neural networks have revolutionized computer vision. High performing deep neural architectures trained for visual recognition tasks have pushed the field towards methods relying on learned image representations instead of hand-crafted ones, in the seek of designing end-to-end learning methods to solve challenging tasks, ranging from long-lasting ones such as image classification to newly emerging tasks like image captioning.\ 

As this thesis is framed in the context of the rapid evolution of computer vision, we present contributions that are aligned with three major changes in paradigm that the field has recently experienced, namely 1) the power of re-utilizing deep features from pre-trained neural networks for different tasks, 2) the advantage of formulating problems with end-to-end solutions given enough training data, and 3) the growing interest of describing visual data with natural language rather than pre-defined categorical label spaces, which can in turn enable visual understanding beyond scene recognition.

The first part of the thesis is dedicated to the problem of visual instance search, where we particularly focus on obtaining meaningful and discriminative image representations which allow efficient and effective retrieval of similar images given a visual query. Contributions in this part of the thesis involve the construction of sparse Bag-of-Words image representations from convolutional features from a pre-trained image classification neural network, and an analysis of the advantages of fine-tuning a pre-trained object detection network using query images as training data.

The second part of the thesis presents contributions to the problem of image-to-set prediction, understood as the task of predicting a variable-sized collection of unordered elements for an input image. We conduct a thorough analysis of current methods for multi-label image classification, which are able to solve the task in an end-to-end manner by simultaneously estimating both the label distribution and the set cardinality. Further, we extend the analysis of set prediction methods to semantic instance segmentation, and present an end-to-end recurrent model that is able to predict sets of objects (binary masks and categorical labels) in a sequential manner.

Finally, the third part of the dissertation takes insights learned in the previous two parts in order to present deep learning solutions to connect images with natural language in the context of cooking recipes and food images. First, we propose a retrieval-based solution in which the written recipe and the image are encoded into compact representations that allow the retrieval of one given the other. Second, as an alternative to the retrieval approach, we propose a generative model to predict recipes directly from food images, which first predicts ingredients as sets and subsequently generates the rest of the recipe one word at a time by conditioning both on the image and the predicted ingredients.

}, url = {https://www.tdx.cat/handle/10803/667162}, author = {Amaia Salvador}, editor = {Xavier Gir{\'o}-i-Nieto and Marqu{\'e}s, F.} } @conference {cSalvadorf, title = {Inverse Cooking: Recipe Generation from Food Images}, booktitle = {CVPR}, year = {2019}, month = {06/2019}, publisher = {OpenCVF / IEEE}, organization = {OpenCVF / IEEE}, address = {Long Beach, CA, USA}, abstract = {

People enjoy food photography because they appreciate food. Behind each meal there is a story described in a complex recipe and, unfortunately, by simply looking at a food image we do not have access to its preparation process. Therefore, in this paper we introduce an inverse cooking system that recreates cooking recipes given food images. Our system predicts ingredients as sets by means of a novel architecture, modeling their dependencies without imposing any order, and then generates cooking instructions by attending to both image and its inferred ingredients simultaneously. We extensively evaluate the whole system on the large-scale Recipe1M dataset and show that (1) we improve performance w.r.t. previous baselines for ingredient prediction; (2) we are able to obtain high quality recipes by leveraging both image and ingredients; (3) our system is able to produce more compelling recipes than retrieval-based approaches according to human judgment.

}, url = {http://openaccess.thecvf.com/content_CVPR_2019/html/Salvador_Inverse_Cooking_Recipe_Generation_From_Food_Images_CVPR_2019_paper.html}, author = {Amaia Salvador and Drozdzal, Michal and Xavier Gir{\'o}-i-Nieto and Romero, Adriana} } @conference {cVenturaa, title = {RVOS: End-to-End Recurrent Network for Video Object Segmentation}, booktitle = {CVPR}, year = {2019}, month = {06/2019}, publisher = {OpenCVF / IEEE}, organization = {OpenCVF / IEEE}, address = {Long Beach, CA, USA}, abstract = {

Multiple object video object segmentation is a challenging task, specially for the zero-shot case, when no object mask is given at the initial frame and the model has to find the objects to be segmented along the sequence. In our work, we propose RVOS, a recurrent network that is fully end-to-end trainable for multiple object video object segmentation, with a recurrence module working on two different domains: (i) the spatial, which allows to discover the different object instances within a frame, and (ii) the temporal, which allows to keep the coherence of the segmented objects along time. We train RVOS for zero-shot video object segmentation and are the first ones to report quantitative results for DAVIS-2017 and YouTube-VOS benchmarks. Further, we adapt RVOS for one-shot video object segmentation by using the masks obtained in previous time-steps as inputs to be processed by the recurrent module. Our model reaches comparable results to state-of-the-art techniques in YouTube-VOS benchmark and outperforms all previous video object segmentation methods not using online learning in the DAVIS-2017 benchmark.

RVOS: End-to-End Recurrent Network for Video Object Segmentation (CVPR 2019) from Universitat Polit{\`e}cnica de Catalunya



}, url = {http://openaccess.thecvf.com/content_CVPR_2019/html/Ventura_RVOS_End-To-End_Recurrent_Network_for_Video_Object_Segmentation_CVPR_2019_paper.html}, author = {Ventura, C. and M{\'\i}riam Bellver and Girbau, A. and Amaia Salvador and Marqu{\'e}s, F. and Xavier Gir{\'o}-i-Nieto} } @conference {cDuartea, title = {Wav2Pix: Speech-conditioned Face Generation using Generative Adversarial Networks}, booktitle = {ICASSP}, year = {2019}, month = {05/2019}, publisher = {IEEE}, organization = {IEEE}, address = {Brighton, UK}, abstract = {

Speech is a rich biometric signal that contains information\ about the identity, gender and emotional state of the speaker.\ In this work, we explore its potential to generate face images\ of a speaker by conditioning a Generative Adversarial\ Network (GAN) with raw speech input. We propose a deep\ neural network that is trained from scratch in an end-to-end\ fashion, generating a face directly from the raw speech waveform\ without any additional identity information (e.g reference\ image or one-hot encoding). Our model is trained in a\ self-supervised fashion by exploiting the audio and visual signals\ naturally aligned in videos. With the purpose of training\ from video data, we present a novel dataset collected for this\ work, with high-quality videos of ten youtubers with notable\ expressiveness in both the speech and visual signals.

}, doi = {10.1109/ICASSP.2019.8682970}, url = {http://hdl.handle.net/2117/167073}, author = {Amanda Duarte and Rold{\'a}n, Francisco and Tubau, Miquel and Escur, Janna and Pascual-deLaPuente, Santiago and Amaia Salvador and Mohedano, Eva and McGuinness, Kevin and Jordi Torres and Xavier Gir{\'o}-i-Nieto} } @conference {cSuris, title = {Cross-modal Embeddings for Video and Audio Retrieval}, booktitle = {ECCV 2018 Women in Computer Vision Workshop}, year = {2018}, month = {09/2018}, publisher = {Springer}, organization = {Springer}, address = {Munich, Germany}, abstract = {

The increasing amount of online videos brings several opportunities for training self-supervised neural networks. The creation of large scale datasets of videos such as the YouTube-8M allows us to deal with this large amount of data in manageable way. In this work, we find new ways of exploiting this dataset by taking advantage of the multi-modal information it provides. By means of a neural network, we are able to create links between audio and visual documents, by projecting them into a common region of the feature space, obtaining joint audio-visual embeddings. These links are used to retrieve audio samples that fit well to a given silent video, and also to retrieve images that match a given a query audio. The results in terms of Recall@K obtained over a subset of YouTube-8M videos show the potential of this unsupervised approach for cross-modal feature learning. We train embeddings for both scales and assess their quality in a retrieval problem, formulated as using the feature extracted from one modality to retrieve the most similar videos based on the features computed in the other modality.

Cross Modal Embeddings for Video and Audio Retrieval $\#$WiCV18 from Universitat Polit{\`e}cnica de Catalunya
}, isbn = {978-3-030-11018-5}, doi = {10.1007/978-3-030-11018-5_62}, url = {https://doi.org/10.1007/978-3-030-11018-5_62}, author = {Sur{\'\i}s, D{\'\i}dac and Amanda Duarte and Amaia Salvador and Jordi Torres and Xavier Gir{\'o}-i-Nieto} } @conference {cSalvadore, title = {Recurrent Neural Networks for Semantic Instance Segmentation}, booktitle = {CVPR 2018 DeepVision Workshop}, year = {2018}, month = {06/2018}, abstract = {

We present a recurrent model for semantic instance segmentation that sequentially generates binary masks and their associated class probabilities for every object in an image. Our proposed system is trainable end-to-end from an input image to a sequence of labeled masks and, compared to methods relying on object proposals, does not require post-processing steps on its output. We study the suitability of our recurrent model on three different instance segmentation benchmarks, namely Pascal VOC 2012, CVPPP Plant Leaf Segmentation and Cityscapes. Further, we analyze the object sorting patterns generated by our model and observe that it learns to follow a consistent pattern, which correlates with the activations learned in the encoder part of our network.

}, author = {Amaia Salvador and M{\'\i}riam Bellver and Baradad, Manel and V{\'\i}ctor Campos and Marqu{\'e}s, F. and Jordi Torres and Xavier Gir{\'o}-i-Nieto} } @conference {cSalvadord, title = {Recurrent Neural Networks for Semantic Instance Segmentation}, booktitle = {ECCV 2018 Women in Computer Vision (WiCV) Workshop}, year = {2018}, month = {12/2017}, abstract = {

We present a recurrent model for semantic instance segmentation that sequentially generates pairs of masks and their associated class probabilities for every object in an image. Our proposed system is trainable end-to-end, does not require post-processing steps on its output and is conceptually simpler than current methods relying on object proposals. We observe that our model learns to follow a consistent pattern to generate object sequences, which correlates with the activations learned in the encoder part of our network. We achieve competitive results on three different instance segmentation benchmarks (Pascal VOC 2012, Cityscapes and CVPPP Plant Leaf Segmentation).

Recurrent Neural Networks for Semantic Instance Segmentation from Universitat Polit{\`e}cnica de Catalunya
}, url = {https://imatge-upc.github.io/rsis/}, author = {Amaia Salvador and M{\'\i}riam Bellver and Baradad, Manel and V{\'\i}ctor Campos and Marqu{\'e}s, F. and Jordi Torres and Xavier Gir{\'o}-i-Nieto} } @mastersthesis {xRoldana, title = {Speech-conditioned Face Generation with Deep Adversarial Networks}, year = {2018}, abstract = {

Image synthesis have been a trending task for the AI community in recent years. Many works have shown the potential of Generative Adversarial Networks (GANs) to deal with tasks such as text or audio to image synthesis. In particular, recent advances in deep learning using audio have inspired many works involving both visual and auditory information. In this work we propose a face synthesis method using audio and/or language representations as inputs. Furthermore, a dataset which relates speech utterances with a face and an identity has been built, fitting for other tasks apart from face synthesis such as speaker recognition or voice conversion.

Speech Conditioned Face Generation with Deep Adversarial Networks from Universitat Polit{\`e}cnica de Catalunya
}, author = {Rold{\'a}n, Francisco}, editor = {Pascual-deLaPuente, Santiago and Amaia Salvador and McGuinness, Kevin and Xavier Gir{\'o}-i-Nieto} } @article {xSalvador17, title = {Artificial intelligence suggests recipes based on food photos}, volume = {2017}, year = {2017}, publisher = {MIT News}, address = {Boston}, abstract = {

Given a still image of a dish filled with food, CSAIL team{\textquoteright}s deep-learning algorithm recommends ingredients and recipes.

}, url = {Artificial intelligence suggests recipes based on food photos}, author = {Amaia Salvador} } @conference {cSalvadorc, title = {Learning Cross-modal Embeddings for Cooking Recipes and Food Images}, booktitle = {CVPR}, year = {2017}, month = {03/2017}, publisher = {CVF / IEEE}, organization = {CVF / IEEE}, address = {Honolulu, Hawaii, USA}, abstract = {

In this paper, we introduce Recipe1M, a new large-scale, structured corpus of over 1m cooking recipes and 800k food images. As the largest publicly available collection of recipe data, Recipe1M affords the ability to train high-capacity models on aligned, multi-modal data. Using these data, we train a neural network to find a joint embedding of recipes and images that yields impressive results on an image-recipe retrieval task. Additionally, we demonstrate that regularization via the addition of a high-level classification objective both improves retrieval performance to rival that of humans and enables semantic vector arithmetic. We postulate that these embeddings will provide a basis for further exploration of the Recipe1M dataset and food and cooking in general.

[Project page]

In the news:

}, doi = {10.1109/CVPR.2017.327}, url = {http://openaccess.thecvf.com/content_cvpr_2017/html/Salvador_Learning_Cross-Modal_Embeddings_CVPR_2017_paper.html}, author = {Amaia Salvador and Hynes, Nicholas and Aytar, Yusuf and Marin, Javier and Ofli, Ferda and Weber, Ingmar and Torralba, Antonio} } @article {xSalvador17b, title = {MIT is building a system that can identify a recipe using pictures of food}, year = {2017}, publisher = {Techcrunch}, abstract = {

Mastering the ability to distinguish hot dogs from not hot dogs is truly one of mankind{\textquoteright}s greatest achievements. Attempting to one-up that sort of masterstroke would surely be flying too close to the sun {\textemdash} but researchers at MIT are trying nonetheless.

}, url = {https://techcrunch.com/2017/07/20/mit-is-building-a-system-that-can-identify-a-recipe-using-pictures-of-food/}, author = {Amaia Salvador} } @inbook {bMohedano17, title = {Object Retrieval with Deep Convolutional Features}, booktitle = {Deep Learning for Image Processing Applications}, volume = {31}, number = {Advances in Parallel Computing}, year = {2017}, publisher = {IOS Press}, organization = {IOS Press}, address = {Amsterdam, The Netherlands}, abstract = {

Image representations extracted from convolutional neural networks (CNNs) outdo hand-crafted features in several computer vision tasks, such as visual image retrieval. This chapter recommends a simple pipeline for encoding the local activations of a convolutional layer of a pretrained CNN utilizing the well-known Bag of Words (BoW) aggregation scheme and called bag of local convolutional features (BLCF). Matching each local array of activations in a convolutional layer to a visual word results in an assignment map, which is a compact representation relating regions of an image with a visual word. We use the assignment map for fast spatial reranking, finding object localizations that are used for query expansion. We show the suitability of the BoW representation based on local CNN features for image retrieval, attaining state-of-the-art performance on the Oxford and Paris buildings benchmarks. We demonstrate that the BLCF system outperforms the latest procedures using sum pooling for a subgroup of the challenging TRECVid INS benchmark according to the mean Average Precision (mAP) metric.

}, issn = {978-1-61499-822-8 }, doi = {10.3233/978-1-61499-822-8-137}, url = {http://ebooks.iospress.nl/volumearticle/48028}, author = {Mohedano, Eva and Amaia Salvador and McGuinness, Kevin and Xavier Gir{\'o}-i-Nieto and O{\textquoteright}Connor, N. and Marqu{\'e}s, F.} } @conference {xSalvadora, title = {Recurrent Semantic Instance Segmentation}, booktitle = {NIPS 2017 Women in Machine Learning Workshop (WiML)}, year = {2017}, month = {12/2017}, publisher = {NIPS 2017 Women in Machine Learning Workshop}, organization = {NIPS 2017 Women in Machine Learning Workshop}, address = {Long Beach, CA, USA}, abstract = {

We present a recurrent model for end-to-end instance-aware semantic segmentation that is able to sequentially generate pairs of masks and class predictions. Our proposed system is trainable end-to-end for instance segmentation, does not require further post-processing steps on its output and is conceptually simpler than current methods relying on object proposals. While recent works have proposed recurrent architectures for instance segmentation, these are trained and evaluated for a single category.

Our model is composed of a series of Convolutional LSTMs that are applied in chain with upsampling layers in between to predict a sequence of binary masks and associated class probabilities. Skip connections are incorporated in our model by concatenating the output of the corresponding convolutional layer in the base model with the upsampled output of the ConvLSTM. Binary masks are finally obtained with a 1x1 convolution with sigmoid activation. We concatenate the side outputs of all ConvLSTM layers and apply a per-channel max-pooling operation followed by a single fully-connected layer with softmax activation to obtain the category for each predicted mask.

We train and evaluate our models with the Pascal VOC 2012 dataset. Future work will aim at analyzing and understanding the behavior of the network on other datasets, comparing the system with state of the art solutions and study the relationship of the learned object discovery patterns of our model with those of humans.

}, author = {Amaia Salvador and Baradad, Manel and Xavier Gir{\'o}-i-Nieto and Marqu{\'e}s, F.} } @article {xSalvador17a, title = {Snap a photo, get a recipe: pic2recipe uses AI to predict food ingredients}, year = {2017}, publisher = {Digital Trends}, abstract = {

No recipe, no problem. Using image recognition, we can now learn how to cook something just by analyzing a photo.

}, url = {https://www.digitaltrends.com/photography/pics2recipe-mit-research/}, author = {Amaia Salvador} } @conference {xDuarte, title = {Temporal-aware Cross-modal Embeddings for Video and Audio Retrieval}, booktitle = {NIPS 2017 Women in Machine Learning Workshop (WiML)}, year = {2017}, month = {12/2017}, publisher = {NIPS 2017 Women in Machine Learning Workshop}, organization = {NIPS 2017 Women in Machine Learning Workshop}, address = {Long Beach, CA, USA}, abstract = {

The increasing amount of videos online brings several opportunities for training self-supervised neural networks. In this work, we explore cross-modal embeddings between audio and vision by exploiting their alignment on YouTube videos.

Joint audio-visual embeddings allow creating links between audio and visual documents by projecting them to a common region of the feature space. They can be applied to enriching radio broadcasts with images, finding soundtracks for user-generated videos or simply enriching a topic search with both audio and video documents.

The idea of creating a joint embedding space across modalities has being exploited by other areas [3, 4]. However, joint representation between the video frames and its audio have yet to be fully exploited. A similar approach to the proposed one was [2], where a soundtrack was retrieved to match a music video. However, this work did not target a synchronization between both modalities.

We aim at training a temporal-aware embedding which can align both audio and visual tracks. We use the visual and audio features provided in the YouTube-8M dataset [1]. The dataset includes features at both the clip and frame (temporal window) level. We train embeddings for both scales and assess their quality in a retrieval problem, formulated as using the feature extracted from one modality to retrieve the most similar videos based on the features computed in the other modality.

We aim at not only finding related documents, but synchronize both sequences. The alignment between the two sequences will rely on computing temporal-aware features with recurrent neural networks at different scales. At retrieval time, different scales will be assessed and results evaluated both with ranking metrics and Amazon Mechanical Turk.

References

[1] Sami Abu-El-Haija, Nisarg Kothari, Joonseok Lee, Paul Natsev, George Toderici, Balakrishnan Varadarajan, and Sudheendra Vijayanarasimhan. Youtube-8m: A large-scale video classification benchmark. arXiv preprint arXiv:1609.08675, 2016.

[2] Sungeun Hong, Woobin Im, and Hyun S Yang. Deep learning for content-based, cross-modal retrieval of videos and music. arXiv preprint arXiv:1704.06761, 2017.

[3] Amaia Salvador, Nicholas Hynes, Yusuf Aytar, Javier Marin, Ferda Ofli, Ingmar Weber, and Antonio Torralba. Learning cross-modal embeddings for cooking recipes and food images. In CVPR, 2017.

[4] Liwei Wang, Yin Li, and Svetlana Lazebnik. Learning deep structure-preserving image-text embeddings. In CVPR, 2016.

}, author = {Amanda Duarte and Sur{\'\i}s, D{\'\i}dac and Amaia Salvador and Jordi Torres and Xavier Gir{\'o}-i-Nieto} } @article {a, title = {Assessment of Crowdsourcing and Gamification Loss in User-Assisted Object Segmentation}, journal = {Multimedia Tools and Applications}, volume = {23}, year = {2016}, month = {11/2016}, chapter = {15901-15928}, abstract = {

There has been a growing interest in applying human computation -- \ particularly crowdsourcing techniques -- to assist in the solution of multimedia, image processing, and computer vision problems which are still too difficult to solve using fully automatic algorithms, and yet relatively easy for humans.

In this paper we focus on a specific problem -- object segmentation within color images -- and compare different solutions which combine color image segmentation algorithms with human efforts, either in the form of an explicit interactive segmentation task or through an implicit collection of valuable human traces with a game.\ We use Click{\textquoteright}n{\textquoteright}Cut, a friendly, web-based, interactive segmentation tool that allows segmentation tasks to be assigned to many users, and Ask{\textquoteright}nSeek, a game with a purpose designed for object detection and segmentation.

The two main contributions of this paper are: (i) We use the results of Click{\textquoteright}n{\textquoteright}Cut campaigns with different groups of users to examine and quantify the crowdsourcing loss incurred when an interactive segmentation task is assigned to paid crowd-workers, comparing their results to the ones obtained when computer vision experts are asked to perform the same tasks. (ii) Since interactive segmentation tasks are inherently tedious and prone to fatigue, we\ compare the quality \ of the results obtained with Click{\textquoteright}n{\textquoteright}Cut with the ones obtained using a (fun, interactive, and potentially less tedious) game designed for the same purpose. We call this contribution the assessment of the gamification loss, since it refers to how much quality of segmentation results may be lost when we switch to a game-based approach to the same task.\ 

We demonstrate that the crowdsourcing loss is significant when using all the data points from workers, but decreases substantially (and becomes comparable to the quality of expert users performing similar tasks) after performing a modest amount of data analysis and filtering out of users whose data are clearly not useful. We also show that -- on the other hand -- the gamification loss is significantly more severe: the quality of the results drops roughly by half when switching from a focused (yet tedious) task to a more fun and relaxed game environment.\ 

}, keywords = {Crowdsourcing, GWAP, Object detection, Object segmentation, Serious games}, issn = {1573-7721}, doi = {10.1007/s11042-015-2897-6}, url = {http://dx.doi.org/10.1007/s11042-015-2897-6}, author = {Carlier, Axel and Amaia Salvador and Cabezas, Ferran and Xavier Gir{\'o}-i-Nieto and Charvillat, Vincent and Marques, Oge} } @conference {cMohedanoa, title = {Bags of Local Convolutional Features for Scalable Instance Search}, booktitle = {ACM International Conference on Multimedia Retrieval (ICMR)}, year = {2016}, month = {06/2016}, publisher = {ACM}, organization = {ACM}, address = {New York City, NY; USA}, abstract = {

Image representations extracted from convolutional neural networks (CNNs) have been shown to outperform hand-crafted features in multiple computer vision tasks, such as visual image retrieval. This work proposes a simple pipeline for encoding the local activations of a convolutional layer of a pre-trained CNN using the well-known bag of words aggregation scheme (BoW). Assigning each local array of activations in a convolutional layer to a visual word produces an \textit{assignment map}, a compact representation that relates regions of an image with a visual word. We use the assignment map for fast spatial reranking, obtaining object localizations that are used for query expansion. We demonstrate the suitability of the Bag of Words representation based on local CNN features for image retrieval, achieving state-of-the-art performance on the Oxford and Paris buildings benchmarks. We show that our proposed system for CNN feature aggregation with BoW outperforms state-of-the-art techniques using sum pooling at a subset of the challenging TRECVid INS benchmark.

Best poster award at ACM ICMR 2016

Preprint on arXiv

Project page with source code

Post on GitXiv

Overall acceptance rate in ICMR 2016: 30\%\ 

2016-05-Seminar-AmaiaSalvador-DeepVision from Image Processing Group on Vimeo.

Convolutional Features for Instance Search from Xavier Giro
}, keywords = {Bag of Words, Convolutional Neural Networks, Instance Retrieval}, isbn = {978-1-4503-4359-6}, doi = {http://dx.doi.org/10.1145/2911996.2912061}, url = {http://dx.doi.org/10.1145/2911996.2912061}, author = {Mohedano, Eva and Amaia Salvador and McGuinness, Kevin and Xavier Gir{\'o}-i-Nieto and O{\textquoteright}Connor, N. and Marqu{\'e}s, F.} } @conference {cSalvadorb, title = {Faster R-CNN Features for Instance Search}, booktitle = {CVPR Workshop Deep Vision}, year = {2016}, month = {06/2016}, abstract = {

Image representations derived from pre-trained Convolutional Neural Networks (CNNs) have become the new state of the art in computer vision tasks such as instance retrieval. This work explores the suitability for instance retrieval of image- and region-wise representations pooled from an object detection CNN such as Faster R-CNN. We take advantage of the object proposals learned by a Region Proposal Network (RPN) and their associated CNN features to build an instance search pipeline composed of a first filtering stage followed by a spatial reranking. We further investigate the suitability of Faster R-CNN features when the network is fine-tuned for the same objects one wants to retrieve. We assess the performance of our proposed system with the Oxford Buildings 5k, Paris Buildings 6k and a subset of TRECVid Instance Search 2013, achieving competitive results.

2016-05-Seminar-AmaiaSalvador-DeepVision from Image Processing Group on Vimeo.

Convolutional Features for Instance Search from Xavier Giro
}, url = {http://www.cv-foundation.org/openaccess/content_cvpr_2016_workshops/w12/papers/Salvador_Faster_R-CNN_Features_CVPR_2016_paper.pdf}, author = {Amaia Salvador and Xavier Gir{\'o}-i-Nieto and Marqu{\'e}s, F. and Satoh, Shin{\textquoteright}ichi} } @mastersthesis {xFerri16, title = {Object Tracking in Video with TensorFlow}, year = {2016}, abstract = {

[Project repo]

[Additional repo for setting up the environment]

}, author = {Ferri, Andrea}, editor = {Xavier Gir{\'o}-i-Nieto and Jordi Torres and Amaia Salvador} } @conference {cMontes, title = {Temporal Activity Detection in Untrimmed Videos with Recurrent Neural Networks}, booktitle = {1st NIPS Workshop on Large Scale Computer Vision Systems 2016}, year = {2016}, month = {12/2016}, abstract = {

This work proposes a simple pipeline to classify and temporally localize activities in untrimmed videos. Our system uses features from a 3D Convolutional Neural Network (C3D) as input to train a a recurrent neural network (RNN) that learns to classify video clips of 16 frames. After clip prediction, we post-process the output of the RNN to assign a single activity label to each video, and determine the temporal boundaries of the activity within the video. We show how our system can achieve competitive results in both tasks with a simple architecture. We evaluate our method in the ActivityNet Challenge 2016, achieving a 0.5874 mAP and a 0.2237 mAP in the classification and detection tasks, respectively. Our code and models are publicly available at:\ https://imatge-upc.github.io/activitynet-2016-cvprw/

}, author = {Montes, Alberto and Amaia Salvador and Pascual-deLaPuente, Santiago and Xavier Gir{\'o}-i-Nieto} } @mastersthesis {xMontes, title = {Temporal Activity Detection in Untrimmed Videos with Recurrent Neural Networks}, year = {2016}, abstract = {

Advisors: Amaia Salvador and Xavier Gir{\'o}-i-Nieto.

Studies: Bachelor degree in Science and Telecommunication Technologies Engineering at\ Telecom BCN-ETSETB\ from the Technical University of Catalonia (UPC)

Grade: A (9.8/10.0)

This thesis explore different approaches using Convolutional and Recurrent Neural Networks to classify and temporally localize activities on videos, furthermore an implementation to achieve it has been proposed.

As the first step, features have been extracted from video frames using an state of the art 3D Convolutional Neural Network. This features are fed in a recurrent neural network that solves the activity classification and temporally location tasks in a simple and flexible way.

Different architectures and configurations have been tested in order to achieve the best performance and learning of the video dataset provided. In addition it has been studied different kind of post processing over the trained network{\textquoteright}s output to achieve a better results on the temporally localization of activities on the videos.

The results provided by the neural network developed in this thesis have been submitted to the ActivityNet Challenge 2016 of the CVPR, achieving competitive results using a simple and flexible architecture.

Temporal Activity Detection in Untrimmed Videos with Recurrent Neural Networks from Xavier Giro
}, keywords = {deep learning, neural networks, videos}, author = {Montes, Alberto}, editor = {Amaia Salvador and Xavier Gir{\'o}-i-Nieto} } @mastersthesis {xCabezas, title = {Co-filtering human interaction and object segmentation}, year = {2015}, abstract = {

Advisors: Axel Carlier and Vincent Charvillat (ENSEEIHT-Universit{\'e} de Toulouse) / Amaia Salvador and\ Xavier Gir{\'o}-i-Nieto\ (UPC)\ 

Degree: Audiovisual Engineering (4 years) at\ Telecom BCN-ETSETB\ (UPC)

Grade: A with honors (9.6/10)

For so many years the problem of object segmentation has been present in image processing field. Click{\textquoteright}n{\textquoteright}Cut, an already existing web tool for interactive object segmentation, helps us to obtain segmentations of the objects by clicking in green (foreground clicks) inside the object to segment, and in red(background clicks) outside the object to segment. However, the behaviour of all human in front of this web tool is not equal. For this reason, it can be possible that these human interactions do not help us to obtain a good object segmentation as would be the result of a bad human interaction. The main aim of this project is to implement some techniques that allow us to treat with these bad human interactions in order to obtain the best object segmentation possible.

Co-filtering human interaction and object segmentation from Xavi Gir{\'o}
}, url = {http://hdl.handle.net/2099.1/25821}, author = {Cabezas, Ferran}, editor = {Carlier, Axel and Amaia Salvador and Xavier Gir{\'o}-i-Nieto and Charvillat, Vincent} } @conference {cSalvadora, title = {Cultural Event Recognition with Visual ConvNets and Temporal Models}, booktitle = {CVPR ChaLearn Looking at People Workshop 2015}, year = {2015}, month = {06/2015}, abstract = {

This paper presents our contribution to the ChaLearn Challenge 2015 on Cultural Event Classification. The challenge in this task is to automatically classify images from 50 different cultural events. Our solution is based on the combination of visual features extracted from convolutional neural networks with temporal information using a hierarchical classifier scheme. We extract visual features from the last three fully connected layers of both CaffeNet (pretrained with ImageNet) and our fine tuned version for the ChaLearn challenge. We propose a late fusion strategy that trains a separate low-level SVM on each of the extracted neural codes. The class predictions of the low-level SVMs form the input to a higher level SVM, which gives the final event scores. We achieve our best result by adding a temporal refinement step into our classification scheme, which is applied directly to the output of each low-level SVM. Our approach penalizes high classification scores based on visual features when their time stamp does not match well an event-specific temporal distribution learned from the training and validation data. Our system achieved the second best result in the \ ChaLearn Challenge 2015 on Cultural Event Classification with a mean average precision of 0.767 on the test set.

[Preprint in arXiv][Workshop site][Slides on GDrive]

Cultural Event Recognition with Visual ConvNets and Temporal Models from Xavier Giro
}, url = {http://www.cv-foundation.org/openaccess/content_cvpr_workshops_2015/W09/papers/Salvador_Cultural_Event_Recognition_2015_CVPR_paper.pdf}, author = {Amaia Salvador and Zeppelzauer, Matthias and Manchon-Vizuete, Daniel and Calafell, Andrea and Xavier Gir{\'o}-i-Nieto} } @conference {cCampos-Camunez, title = {Diving Deep into Sentiment: Understanding Fine-tuned CNNs for Visual Sentiment Prediction}, booktitle = {1st International Workshop on Affect and Sentiment in Multimedia}, year = {2015}, month = {10/2015}, publisher = {ACM}, organization = {ACM}, address = {Brisbane, Australia}, abstract = {

Visual media are powerful means of expressing emotions and sentiments. The constant generation of new content in social networks highlights the need of automated visual sentiment analysis tools. While Convolutional Neural Networks (CNNs) have established a new state-of-the-art in several vision problems, their application to the task of sentiment analysis is mostly unexplored and there are few studies regarding how to design CNNs for this purpose. In this work, we study the suitability of fine-tuning a CNN for visual sentiment prediction as well as explore performance boosting techniques within this deep learning setting. Finally, we provide a deep-dive analysis into a benchmark, state-of-the-art network architecture to gain insight about how to design patterns for CNNs on the task of visual sentiment prediction.

Acceptance rate at the ASM workshop{\textquoteright}15: 56\% (9/16) [source]

Diving deep into sentiment: Understanding fine-tuned CNNs for visual sentiment prediction from Xavier Giro

Visual Sentiment Detection by Deep CNNs $\#$acmmm15 @DocXavi @mad_astronaut @brendanjou - great paper/analysis! pic.twitter.com/RcGchcPA9f

{\textemdash} Damian Borth (@damianborth) October 30, 2015
}, doi = {10.1145/2813524.2813530}, url = {http://arxiv.org/abs/1508.05056}, author = {V{\'\i}ctor Campos and Amaia Salvador and Xavier Gir{\'o}-i-Nieto and Jou, Brendan} } @conference {cMohedano, title = {Exploring EEG for Object Detection and Retrieval}, booktitle = {ACM International Conference on Multimedia Retrieval (ICMR) }, year = {2015}, address = {Shanghai, China}, abstract = {

This paper explores the potential for using Brain Computer Interfaces (BCI) as a relevance feedback mechanism in content-based image retrieval. We investigate if it is possible to capture useful EEG signals to detect if relevant objects are present in a dataset of realistic and complex images. \ We perform several experiments using a rapid serial visual presentation (RSVP) of images at different rates (5Hz and 10Hz) on 8 users with different degrees of familiarization with BCI and the dataset. We then use the feedback from the BCI and mouse-based interfaces to retrieve objects in a subset of TRECVid images. We show that it is indeed possible detect such objects in complex images and, also, that users with previous knowledge on the dataset or experience with the RSVP outperform others. When the users have limited time to annotate the images (100 seconds in our experiments) both interfaces are comparable in performance. Comparing our best users in a retrieval task, we found that EEG-based relevance feedback outperforms mouse-based feedback. The realistic and complex image dataset differentiates our work from previous studies on EEG for image retrieval.

[Extended version in arXiv:1504.02356]

[ACM ICMR 2015 website]

Overall acceptance rate: 33\% (source)

}, doi = {10.1145/2671188.2749368}, url = {http://arxiv.org/abs/1504.02356}, author = {Mohedano, Eva and Amaia Salvador and Porta, Sergi and Xavier Gir{\'o}-i-Nieto and Kevin McGuinness and Healy, Graham and O{\textquoteright}Connor, N.} } @mastersthesis {xCalafell-Oros, title = {Fine-tuning a Convolutional Network for Cultural Event Recognition}, year = {2015}, abstract = {

Advisors: Amaia Salvador (UPC), Matthias Zeppelzauer (FH St P{\"o}lten), Xavier Gir{\'o}-i-Nieto (UPC)

Studies: Bachelor Degree in Audiovisual Systems Engineering at Telecom BCN-ETSETB from the Technical University of Catalonia (UPC)

Grade: A with honors (10/10)

This thesis explores good practices for improving the performance of an existing convnet trained with a dataset of clean data when an additional dataset of noisy data is available. We develop techniques to clean the noisy data with the help of the clean one, a family of solutions that we will refer to as denoising, and then we explore the best sorting of the clean and noisy datasets during the fine-tuning of a convnet. Then we study strategies to select the subset of images of the clean data that will improve the classification performance, a practice we will efer to as fracking. Next, we determine how many layers are actually better to fine-tune in our convnet, given our amount of data. And finally, we compare the classic convnet architecture where a single network is fine-tuned to solve a multi-class problem with the case of fine-tuning a convnet for binary classification for each considered class.

Fine tuning a convolutional network for cultural event recognition from Xavier Giro

2015-TFG-AndreaCalafell-FineTuningConvolutionalNetworkForCulturalEventRecognition from Image Processing Group on Vimeo.

See https://imatge.upc.edu/web/publications/cultural-event-recognition-visual-convnets-and-temporal-models

}, url = {http://hdl.handle.net/2117/78391}, author = {Calafell, Andrea}, editor = {Amaia Salvador and Zeppelzauer, Matthias and Xavier Gir{\'o}-i-Nieto} } @conference {cMcGuinnessa, title = {Insight DCU at TRECVID 2015}, booktitle = {TRECVID 2015 Workshop}, year = {2015}, month = {11/2015}, publisher = {NIST}, organization = {NIST}, address = {Gaithersburg, MD, USA}, abstract = {

Insight-DCU participated in the instance search (INS), semantic indexing (SIN), and localization tasks (LOC) this year.

In the INS task we used deep convolutional network features trained on external data and the query data for this year to train our system. We submitted four runs, three based on convolutional network features, and one based on SIFT/BoW. F A insightdcu 1 was an automatic run using features from the last convolutional layer of a deep network with bag-of-words encoding and achieved 0.123 mAP. F A insightdcu 2 modi ed the previous run to use re-ranking based on an R-CNN model and achieved 0.111 mAP. I A insightdcu 3, our interactive run, achieved 0.269 mAP. Our SIFT-based run F A insightdcu 2 used weak geometric consistency to improve performance over the previous year to 0.187 mAP. Overall we found that using features from the convolutional layers improved performance over features from the fully connected layers used in previous years, and that weak geometric consistency improves performance for local feature ranking.

In the SIN task we again used convolutional network features, this time netuning a network pretrained on external data for the task. We submitted four runs, 2C D A insightdcu.15 1..4 varying the top-level learning algorithm and use of concept co-occurance. 2C D A insightdcu.15 1 used a linear SVM top-level learner, and achieved 0.63 mAP. Exploiting concept co-occurance improved the accuracy of our logistic regression run 2C D A insightdcu.15 3 from 0.058 mAP to 0.6 2C D A insightdcu.15 3.

Our LOC system used training data from IACC.1.B and features similar to our INS run, but using a VLAD encoding instead of a bag-of-words. Unfortunately there was problem with the run that we are still investigating.

Note: UPC and NII participated only in the INS task of this submission.

}, url = {http://www-nlpir.nist.gov/projects/tvpubs/tv.pubs.15.org.html}, author = {Kevin McGuinness and Mohedano, Eva and Amaia Salvador and Zhang, ZhenXing and Marsden, Mark and Wang, Peng and Jargalsaikhan, Iveel and Antony, Joseph and Xavier Gir{\'o}-i-Nieto and Satoh, Shin{\textquoteright}ichi and O{\textquoteright}Connor, N. and Smeaton, Alan F.} } @mastersthesis {xCampos-Camunez, title = {Layer-wise CNN Surgery for Visual Sentiment Prediction}, year = {2015}, abstract = {

Advisors: Amaia Salvador (UPC), Brendan Jou (Columbia University) and Xavier Gir{\'o}-i-Nieto (UPC)

Visual media are powerful means of expressing emotions and sentiments. The constant generation of new content in social networks highlights the need of automated visual sentiment analysis tools. While Convolutional Neural Networks (CNNs) have established a new state-of-the-art in several vision problems, their application to the task of sentiment analysis is mostly unexplored and there are few studies regarding how to design CNNs for this purpose. In this work, we study the suitability of fine-tuning a CNN for visual sentiment prediction as well as explore performance boosting techniques within this deep learning setting. Finally, we provide a deep-dive analysis into a benchmark, state-of-the-art network architecture to gain insight about how to design patterns for CNNs on the task of visual sentiment prediction.

Layer-wise CNN Surgery for Visual Sentiment Prediction from Xavier Giro

}, author = {V{\'\i}ctor Campos}, editor = {Amaia Salvador and Jou, Brendan and Xavier Gir{\'o}-i-Nieto} } @conference {cNguyen, title = {NII-HITACHI-UIT at TRECVID 2015 Instance Search}, booktitle = {TRECVID 2015 Workshop}, year = {2015}, month = {11/2015}, publisher = {NIST}, organization = {NIST}, address = {Gaithersburg, MD, USA}, abstract = {

In this paper, we propose two methods to improve last year instance search framework. Both of them are based on post processing scheme that try to rerank top K shots returned from BOW model. The rst system is to propose a query-adaptive weighting technique between DPM object detectors score and BOW{\textquoteright}s score. In order to nd a good weight, we use a neural network which learns characteristics of the query including number of features, number of shared words and area of the query topic. The second system combines two state-of-the-art object detectors: DPM and Fast RCNN to estimate object location and similarity score, respectively. The nal score is computed using these components together with BOW based similarity score returned from the baseline system. The experimental results show that our system improved pretty much even with a smaller number of top K input ranked list. Compared to other teams, we got the second place with the same run.

}, url = {http://www-nlpir.nist.gov/projects/tvpubs/tv.pubs.15.org.html}, author = {Nguyen, Vinh-Tiep and Duy-Dinh-Le and Amaia Salvador and Caizhi-Zhu and Nguyen, Dinh-Luan and Tran, Minh-Triet and Ngo Duc, Thanh and Anh Duong, Duc and Satoh, Shin{\textquoteright}ichi and Xavier Gir{\'o}-i-Nieto} } @conference {cCabezas, title = {Quality Control in Crowdsourced Object Segmentation}, booktitle = {IEEE International Conference on Image Processing (ICIP), 2015}, year = {2015}, month = {09/2015}, abstract = {

This paper explores processing techniques to deal with noisy data in crowdsourced object segmentation tasks. We use the data collected with "Click{\textquoteright}n{\textquoteright}Cut", an online interactive segmentation tool, and we perform several experiments towards improving the segmentation results. First, we introduce different superpixel-based techniques to filter users{\textquoteright} traces, and assess their impact on the segmentation result. Second, we present different criteria to detect and discard the traces from potential bad users, resulting in a remarkable increase in performance. Finally, we show a novel superpixel-based segmentation algorithm which does not require any prior filtering and is based on weighting each user{\textquoteright}s contribution according to his/her level of expertise.

Selected among Top 10\% papers in ICIP 2015 based on the reviewer scores and recommendations.

[Preprint on arXiv]

[Related BSc thesis by Ferran Cabezas]

[IEEE ICIP 2015 conference website]

Co-filtering human interaction and object segmentation from Xavier Giro

}, url = {http://arxiv.org/abs/1505.00145}, author = {Cabezas, Ferran and Carlier, Axel and Amaia Salvador and Xavier Gir{\'o}-i-Nieto and Charvillat, Vincent} } @mastersthesis {xPorta, title = {Rapid Serial Visual Presentation for Relevance Feedback in Image Retrieval with EEG Signals}, year = {2015}, abstract = {

Studies: Bachelor degree in Science and Telecommunication Technologies Engineering at\ Telecom BCN-ETSETB\ from the Technical University of Catalonia (UPC)

Grade: A (9/10)

This thesis explores the potential of relevance feedback for image retrieval using EEG signals for human-computer interaction. This project aims at studying the optimal parameters of a rapid serial visual presentation (RSVP) of frames from a video database when the user is searching for an object instance. The simulations reported in this thesis assess the trade-off between using a small or a large amount of images in each RSVP round that captures the user feedback. While short RSVP rounds allow a quick learning of the user intention from the system, RSVP rounds must also be long enough to let users generate the P300 EEG signals which are triggered by relevant images. This work also addresses the problem of how to distribute potential relevant and non-relevant images in a RSVP round to maximize the probabilities of displaying each relevant frame separated at least 1 second from another relevant frame, as this configuration generates a cleaner P300 EEG signal. The presented simulations are based on a realistic set up for video retrieval with a subset of 1,000 frames from the TRECVID 2014 Instance Search task.

}, keywords = {eeg, feedback, image, relevance, retrieval}, author = {Porta, Sergi}, editor = {Amaia Salvador and Mohedano, Eva and Xavier Gir{\'o}-i-Nieto and O{\textquoteright}Connor, N.} } @mastersthesis {xBosch, title = {Region-oriented Convolutional Networks for Object Retrieval}, year = {2015}, abstract = {

Advisors: Amaia Salvador and\ Xavier Gir{\'o}-i-Nieto\ (UPC)\ 

Study program: Engineering on Audiovisual Systems (4 years) at Escola d{\textquoteright}Enginyeria de Terrassa\ (UPC)

Grade: A (9.6/10)

This thesis is framed in the computer vision field, addressing a challenge related to instance search. Instance search consists in searching for occurrences of a certain visual instance on a large collection of visual content, and generating a ranked list of results sorted according to their relevance to a user query. This thesis builds up on existing work presented at the TRECVID Instance Search Task in 2014, and explores the use of local deep learning features extracted from object proposals. The performance of different deep learning architectures (at both global and local scales) is evaluated, and a thorough comparison of them is performed. Secondly, this thesis presents the guidelines to follow in order to fine-tune a convolutional neural network for tasks such as image classification, object detection and semantic segmentation. It does so with the final purpose of fine tuning SDS, a CNN trained for both object detection and semantic segmentation, with the recently released Microsoft COCO dataset.

Region-oriented Convolutional Networks for Object Retrieval from Xavier Giro
}, author = {Fontdevila-Bosch, Eduard}, editor = {Amaia Salvador and Xavier Gir{\'o}-i-Nieto} } @conference {cCarlier, title = {Click{\textquoteright}n{\textquoteright}Cut: Crowdsourced Interactive Segmentation with Object Candidates}, booktitle = {3rd International ACM Workshop on Crowdsourcing for Multimedia (CrowdMM)}, year = {2014}, month = {11/2014}, address = {Orlando, Florida (USA)}, abstract = {

This paper introduces Click{\textquoteright}n{\textquoteright}Cut, a novel web tool for interactive object segmentation addressed to crowdsourcing tasks. Click{\textquoteright}n{\textquoteright}Cut combines bounding boxes and clicks generated by workers to obtain accurate object segmentations. These segmentations are created by combining precomputed object candidates in a light computational fashion that allows an immediate response from the interface. Click{\textquoteright}n{\textquoteright}Cut has been tested with a crowdsourcing campaign to annotate a subset of the Berkeley Segmentation Dataset (BSDS). Results show competitive results with state of the art, especially in time to converge to a high quality segmentation. The data collection campaign included golden standard tests to detect cheaters.

[Related master thesis by Amaia Salvador]

[Related Phd thesis by Axel Carlier]

[CrowdMM website]

}, keywords = {Crowdsourcing, figure-ground segmentation, human computing, object candidates}, doi = {10.1145/2660114.2660125}, url = {http://dx.doi.org/10.1145/2660114.2660125}, author = {Carlier, Axel and Amaia Salvador and Xavier Gir{\'o}-i-Nieto and Marques, Oge and Charvillat, Vincent} } @mastersthesis {xSalvador, title = {Exploiting User Interaction and Object Candidates for Instance Retrieval and Object Segmentation}, year = {2014}, abstract = {

Author: Amaia Salvador-Aguilera

Advisors:\ Xavier Gir{\'o}-i-Nieto\ (UPC) and Kevin McGuinness (Dublin CIty University)

Degree:\ Master in Computer Vision\ (1 year)

Video: Thesis defense

This thesis addresses two of the main challenges nowadays for computer vision: object segmentation and visual instance retrieval. The methodologies proposed to solve both problems are based on the use of object candidates and human computation in the computer vision loop. In the object segmentation side, this work explores how human computation can be useful to achieve better segmentation results, by combining users{\textquoteright} traces with a segmentation algorithm based on object candidates. On the other hand, the instance retrieval problem is also addressed using object candidates to compute local features, and involving the user in the retrieval loop by applying relevance feedback strategies.

}, keywords = {computer Vision, human computing, instance search, object candidates, segmentation}, author = {Amaia Salvador}, editor = {Xavier Gir{\'o}-i-Nieto and Kevin McGuinness} } @conference {cMcGuinness, title = {Insight Centre for Data Analytics (DCU) at TRECVid 2014: Instance Search and Semantic Indexing Tasks}, booktitle = {2014 TRECVID Workshop}, year = {2014}, month = {11/2014}, publisher = {National Institute of Standards and Technology (NIST)}, organization = {National Institute of Standards and Technology (NIST)}, address = {Orlando, Florida (USA)}, abstract = {

Insight-DCU participated in the instance search (INS) and semantic indexing (SIN) tasks in 2014. Two very different approaches were submitted for instance search, one based on features extracted using pre-trained deep convolutional neural networks (CNNs), and another based on local SIFT features, large vocabulary visual bag-of-words aggregation, inverted index-based lookup, and geometric verification on the top-N retrieved results. Two interactive runs and two automatic runs were submitted, the best interactive runs achieved a mAP of 0.135 and the best automatic 0.12. Our semantic indexing runs were based also on using convolutional neural network features, and on Support Vector Machine classifiers with linear and RBF kernels. One run was submitted to the main task, two to the no annotation task, and one to the progress task. Data for the no-annotation task was gathered from Google Images and ImageNet. The main task run has achieved a mAP of 0.086, the best no-annotation runs had a close performance to the main run by achieving a mAP of 0.080, while the progress run had 0.043.

[2014 TREC Video Retrieval Evaluation Notebook Papers and Slides]

Poster

}, url = {http://hdl.handle.net/2117/24915}, author = {Kevin McGuinness and Mohedano, Eva and Zhang, ZhenXing and Hu, Feiyan and Albatal, Rami and Gurrin, Cathal and O{\textquoteright}Connor, N. and Smeaton, Alan F. and Amaia Salvador and Xavier Gir{\'o}-i-Nieto and Ventura, C.} } @mastersthesis {xSalvador13, title = {Crowdsourced Object Segmentation with a Game}, year = {2013}, abstract = {

Co-advised with Axel Carlier (INP Toulouse), Vincent Charvillat\ (INP Toulouse) and Oge Marques (Florida Atlantic University).

Amaia Salvador, "Crowdsourced Object Segmentation with a Game" from Image Processing Group on Vimeo.

}, author = {Amaia Salvador}, editor = {Xavier Gir{\'o}-i-Nieto and Carlier, Axel and Charvillat, Vincent and Marques, Oge} } @conference {cSalvador13 , title = {Crowdsourced Object Segmentation with a Game}, booktitle = {ACM Workshop on Crowdsourcing for Multimedia (CrowdMM)}, year = {2013}, month = {10/2013}, address = {Barcelona}, abstract = {

We introduce a new algorithm for image segmentation based on crowdsourcing through a game : Ask{\textquoteright}nSeek. The game provides information on the objects of an image, under the form of clicks that are either on the object, or on the background. These logs are then used in order to determine the best segmentation for an object among a set of candidates generated by the state-of-the-art CPMC algorithm. We also introduce a simulator that allows the generation of game logs and therefore gives insight about the number of games needed on an image to perform acceptable segmentation.

Amaia Salvador, "Crowdsourced Object Segmentation with a Game" from Xavi Gir{\'o}
}, isbn = {978-1-4503-2396-3}, doi = {http://dx.doi.org/10.1145/2506364.2506367}, url = {http://dx.doi.org/10.1145/2506364.2506367}, author = {Amaia Salvador and Carlier, Axel and Xavier Gir{\'o}-i-Nieto and Marques, Oge and Charvillat, Vincent} }