@conference {cCaselles, title = {SIRA: Relightable Avatars from a Single Image}, booktitle = {Winter Conference on Applications of Computer Vision (WACV)}, year = {2023}, abstract = {

Recovering the geometry of a human head from a single image, while factorizing the materials and illumination is a severely ill-posed problem that requires prior information to be solved. Methods based on 3D Morphable Models (3DMM), and their combination with differentiable renderers, have shown promising results. However, the expressiveness of 3DMMs is limited, and they typically yield over-smoothed and identity-agnostic 3D shapes limited to the face region. Highly accurate full head reconstructions have recently been obtained with neural fields that parameterize the geometry using multilayer perceptrons. The versatility of these representations has also proved effective for disentangling geometry, materials and lighting. However, these methods require several tens of input images. In this paper, we introduce SIRA, a method which, from a single image, reconstructs human head avatars with high fidelity geometry and factorized lights and surface materials. Our key ingredients are two data-driven statistical models based on neural fields that resolve the ambiguities of single-view 3D surface reconstruction and appearance factorization. Experiments show that SIRA obtains state of the art results in 3D head reconstruction while at the same time it successfully disentangles the global illumination, and the diffuse and specular albedos. Furthermore, our reconstructions are amenable to physically-based appearance editing and head model relighting.

}, author = {Caselles, Pol and Ramon, Eduard and Garcia, Jaime and Xavier Gir{\'o}-i-Nieto and Moreno, Francesc and Triginer, Gil} } @phdthesis {dRamon22, title = {Few-shot 3D Reconstruction of Body Parts with Deep Neural Networks}, volume = {Excellent}, year = {2022}, month = {09/2022}, type = {Industrial}, abstract = {

In this thesis, we study the problem of reconstructing objects from a concrete category\ in 3D when few images are available as input, i.e. less than 10. We apply our findings to digitalizing human body parts such as heads and torsos for medical applications. The first part of the thesis explores systems that rely on 3D Morphable Models. When approaching a concrete task, training such systems requires expensive manual hyper-parameter tuning of both the architecture and the loss, which is time consuming. We focus on designing novel losses without hyperparameters and modular architectures that allow to train models without tuning efforts. We also aim at providing a fine alignment between the 3D space and the image space by estimating camera poses with a low re-projection error that further improve the texturing process in 3D modelling applications or the rendering process in augmented reality applications. Our findings lead to systems that are very stable and that naturally scale to different scenes.

While 3D Morphable Models are fast and robust, they are still very limited in terms\ of accuracy and expressiveness, which might be prohibitive for applications that require high fidelity. A promising alternative to 3D Morphable Models are implicit functions,which in combination with differentiable rendering techniques have shown impressive results at reconstructing 3D surfaces. However, the later require large sets of images at test time to obtain satisfactory results. In the second part of the thesis, we propose to use a probabilistic model that represents a distribution of implicit surfaces in combination with a differentiable renderer to reduce the number of images required at test time. The resulting 3D reconstruction system is highly accurate and allows to reconstruct a wide variety of human head shapes when only 3 images are available.

}, author = {Ramon, Eduard}, editor = {Moreno, Francesc and Garcia, Jaime and Xavier Gir{\'o}-i-Nieto} } @conference {cRamonb, title = {H3D-Net: Few-Shot High-Fidelity 3D Head Reconstruction}, booktitle = {International Conference on Computer Vision (ICCV)}, year = {2021}, publisher = {IEEE/CVF}, organization = {IEEE/CVF}, address = {Virtual}, abstract = {

Recent learning approaches that implicitly represent surface geometry using coordinate-based neural representations have shown impressive results in the problem of multi-view 3D reconstruction. The effectiveness of these techniques is, however, subject to the availability of a large number (several tens) of input views of the scene, and computationally demanding optimizations. In this paper, we tackle these limitations for the specific problem of few-shot full 3D head reconstruction, by endowing coordinate-based representations with a probabilistic shape prior that enables faster convergence and better generalization when using few input images (down to three). First, we learn a shape model of 3D heads from thousands of incomplete raw scans using implicit representations. At test time, we jointly overfit two coordinate-based neural networks to the scene, one modeling the geometry and another estimating the surface radiance, using implicit differentiable rendering. We devise a two-stage optimization strategy in which the learned prior is used to initialize and constrain the geometry during an initial optimization phase. Then, the prior is unfrozen and fine-tuned to the scene. By doing this, we achieve high-fidelity head reconstructions, including hair and shoulders, and with a high level of detail that consistently outperforms both state-of-the-art 3D Morphable Models methods in the few-shot scenario, and non-parametric methods when large sets of views are available.

}, author = {Ramon, Eduard and Triginer, Gil and Escur, Janna and Pumarola, Albert and Garcia, Jaime and Xavier Gir{\'o}-i-Nieto and Moreno, Francesc} } @mastersthesis {xEscur, title = {Attention-based multi-view 3D reconstruction models}, year = {2020}, abstract = {

Attention mechanism has been successfully used in multiple tasks in the fields of Computer Vision and Natural Language Processing, but has not ever been applied to 3D reconstruction problems. In this work, we explore the potential of attention in a multi-view 3D face recon- struction pipeline. On one hand, we use spatial attention when extracting the features of the input images, taking advantage of the interpretability it provides us. This allows us to validate the proper behaviour of the model. On the other hand, we want to make this multi-view setup invariant to the order of the input image?s views. To do so, instead of concatenating the fea- tures of the different views, we use part of the Transformer architecture as a symmetric merging function, which is based on a multi-head self-attention mechanism, showing an improvement in the performance.

}, author = {Escur, Janna}, editor = {Ramon, Eduard and Xavier Gir{\'o}-i-Nieto} } @conference {cRamon, title = {Hyperparameter-Free Losses for Model-Based Monocular Reconstruction}, booktitle = {ICCV 2019 Workshop on Geometry Meets Deep Learning}, year = {2019}, month = {11/2019}, publisher = {IEEE / Computer Vision Foundation}, organization = {IEEE / Computer Vision Foundation}, address = {Seoul, South Corea}, abstract = {

This work proposes novel hyperparameter-free losses for single view 3D reconstruction with morphable models (3DMM). We dispense with the hyperparameters used in other works by exploiting geometry, so that the shape of the object and the camera pose are jointly optimized in a sole term expression. This simplification reduces the optimization time and its complexity. Moreover, we propose a novel implicit regularization technique based on random virtual projections that does not require additional 2D or 3D annotations. Our experiments suggest that minimizing a shape reprojection error together with the proposed implicit regularization is especially suitable for applications that require precise alignment between geometry and image spaces, such as augmented reality. We evaluate our losses on a large scale dataset with 3D ground truth and publish our implementations to facilitate reproducibility and public benchmarking in this field.

}, author = {Ramon, Eduard and Ruiz, Guillermo and Batard, Thomas and Xavier Gir{\'o}-i-Nieto} } @conference {cRamona, title = {Multi-View 3D Face Reconstruction in the Wild using Siamese Networks}, booktitle = {ICCV 2019 Workshop on 3D Face Alignment in the Wild Challenge Workshop (3DFAW)}, year = {2019}, month = {11/2019}, publisher = {IEEE/CVF}, organization = {IEEE/CVF}, address = {Seoul, South Corea}, abstract = {

In this work, we present a novel learning based approach\ to reconstruct 3D faces from a single or multiple images.\ Our method uses a simple yet powerful architecture based\ on siamese neural networks that helps to extract relevant\ features from each view while keeping the models small. Instead of minimizing multiple objectives, we propose to simultaneously learn the 3D shape and the individual camera\ poses by using a single term loss based on the reprojection\ error, which generalizes from one to multiple views. This allows to globally optimize the whole scene without having to\ tune any hyperparameters and to achieve low reprojection\ errors, which are important for further texture generation.\ Finally, we train our model on a large scale dataset with\ more than 6,000 facial scans. We report competitive results\ in 3DFAW 2019 challenge, showing the effectiveness of our\ method.

}, author = {Ramon, Eduard and Escur, Janna and Xavier Gir{\'o}-i-Nieto} } @unpublished {cRamon19, title = {Plug-and-Train Loss for Model-Based Single View 3D Reconstruction}, journal = {BMVA Technical Meeting: 3D vision with Deep Learning}, year = {2019}, month = {02/2019}, publisher = {UPC}, address = {London, UK}, abstract = {

Obtaining 3D geometry from images is a well studied problem by the computer vision community. In the concrete case of a single image, a considerable amount of prior knowledge is often required to obtain plausible reconstructions. Recently, deep neural networks in combination with 3D morphable models (3DMM) have been used in order to address the lack of scene information, leading to more accurate results. Nevertheless, the losses employed during the training process are usually a linear combination of terms where the coefficients, also called hyperparameters, must be carefully tuned for each dataset to obtain satisfactory results. In this work we propose a hyperparameters-free loss that exploits the geometry of the problem for learning 3D reconstruction from a single image. The proposed formulation is not dataset dependent, is robust against very large camera poses and jointly optimizes the shape of the object and the camera pose.

Plug and-train Loss for Single View 3D Reconstruction from Universitat Polit{\`e}cnica de Catalunya
}, author = {Ramon, Eduard and Villar, Jordi and Ruiz, Guillermo and Batard, Thomas and Xavier Gir{\'o}-i-Nieto} } @unpublished {xRamon, title = {Deep Learning algorithms for 3D Reconstruction and Simulation of Aesthetic Procedures}, year = {2018}, month = {07/2018}, type = {Phd Thesis Proposal}, abstract = {

3D technology is key for a wide range of industries. Medicine, construction, cinema and many other disciplines can nowadays digitalize the world we perceive using 3D reconstruction algorithms, create new objects by means of 3D printers or analyze the world using 3D detection and segmentation techniques.

These tools are transforming how research and industry problems are addressed. Concretely, in the field of aesthetic surgery, a fluent communication between doctors and patients is crucial in order to maximize the success of the operatories and the satisfaction of the patients. A new trend in the industry is to incorporate 3D technology during the consultation, with the goa lof improving that communication. By reconstructing the body of the patient and simulating aesthetic procedures on it, he or she can realise how a concrete modification would look like when applied to his or her body. It is also beneficial for the physicians, since they can measure the effectiveness of the applied treatments and also convert more consultations into operations due to an increase of confidence in the patient.

This thesis proposal defines the research directions to follow during an industrial doctorate to be developed in Crisalix Labs, in collaboration of the Image Processing Group at the Universitat Polit{\`e}cnica de Catalunya. Industrial doctorates are promoted from the Government of Catalonia to promote the transfer of knowledge from universities to local industry as an element for innovation and technical excellence.

}, author = {Ramon, Eduard}, editor = {Xavier Gir{\'o}-i-Nieto} } @mastersthesis {xRamon13, title = {Algorithms for B wave detection}, year = {2013}, abstract = {

The objective of this Master Thesis was to develop algorithms for B waves detection in ICP. This goal was approached by two different methods that depend basically in the resolution of the acquired ICP. Then, both methods were adapted to work in an ultra-low power microcontroller. The first method works using ICP recorded at 1 Hz and it is based on the Lundberg{\textquoteright}s definition of B wave. A plus of this algorithm is that reduces to the minimum the number of samples per block to classify. The results obtained after testing it using long records of ICP from 27 patients were an accuracy of 89,59\%, a specificity 89,71\% and a sensitivity of 89,16\%. These results did not change when the code was adapted to the microcontroller. The second method requires ICP obtained with a sampling rate of 100 Hz. It is based on the morphology of the pulse waves present in the ICP and caused by the change of blood volume inside the skull with every heartbeat. A total of 1430 blocks of ICP (864 for lack of B wave and 566 for presence of B wave), everyone with duration of 41 seconds, were used to extract 21 features from each one. Then a MLP classifier and a SVM classifier were tested and compared. The best results were obtained by the SVM classifier, reaching an accuracy of 86,37\%, a specificity of 88,09\% and a sensitivity of 83,74\% when all features were used. After adapting the algorithm to the microcontroller the results were nearly the same.

}, url = {http://hdl.handle.net/2099.1/19034}, author = {Ramon, Eduard} }