@phdthesis {dMosella-Montoro23, title = {Graph Convolutional Neural Networks for 3D Data Analysis}, volume = {Phd}, year = {2023}, school = {Universitat Polit{\`e}cnica de Catalunya}, address = {Barcelona}, abstract = {

Deep Learning allows the extraction of complex features directly from raw input data, eliminating the need for hand-crafted features from the classical Machine Learning pipeline. This new paradigm brought a boost in the performance across several domains, including computer vision, natural language processing and audio processing. However, there are still challenges when dealing with unorganized structures. This thesis addresses this challenge using Graph Convolutional Neural Networks, a new set of techniques capable of managing graph structures that can be used for processing 3D data.\ \ 

The first part of the thesis focuses on the Graph Analysis task, in which we study the capabilities of Graph Convolutional Neural Networks to capture the intrinsic geometric information of 3D data. We propose the Attention Graph Convolution layer that learns to infer the kernel used during the convolution, taking into account the particularities of each neighbourhood of the graph. We explore two variants of the Attention Graph Convolution layer, one that explores a residual approach and another one that allows the convolution to combine different neighbourhood domains. Furthermore, we propose a set of 3D pooling layers that mimics the behaviour of the pooling layers found in common 2D Convolutional Neural Networks architectures. Finally, we present a 2D-3D Fusion block capable of merging the 3D geometric information that we get from a Graph Convolutional Neural Network with the texture information obtained by a 2D Convolutional Neural Network. We evaluate the presented contributions on the RGB-D Scene Classification task.\ 

The second part of this thesis focuses on the Node Analysis task, which consists of extracting features on a node level, taking into account the neighbourhood structure. We present the Multi-Aggregator Graph Convolution layer that uses a multiple aggregator approach to better generalize for unseen topologies and learn better local representations. In addition, it reduces the memory footprint with respect to the Attention Graph Convolution layer. Finally, we analyze the capabilities of our proposed Graph Convolution layers to deal with heterogeneous graphs where the nodes of the graph may belong to different modalities. We evaluate the presented contributions with the Computer Graphics process of skinning a character mesh. Specifically, we propose a Two-Stream Graph Neural Network capable of predicting the skinning weights of a 3D character.

}, url = {http://hdl.handle.net/10803/689400}, author = {Mosella-Montoro, Albert}, editor = {Ruiz-Hidalgo, J.} } @conference {cMosella-Montoro22, title = {SkinningNet: Two-Stream Graph Convolutional Neural Network for Skinning Prediction of Synthetic Characters}, booktitle = {IEEE / CVF Computer Vision and Pattern Recognition Conference (CVPR)}, year = {2022}, month = {06/2022}, address = {New Orleans, USA}, abstract = {

This work presents SkinningNet, an end-to-end Two-Stream Graph Neural Network architecture that computes skinning weights from an input mesh and its associated skeleton, without making any assumptions on shape class and structure of the provided mesh. Whereas previous methods pre-compute handcrafted features that relate the mesh and the skeleton or assume a fixed topology of the skeleton, the proposed method extracts this information in an end-to-end learnable fashion by jointly learning the best relationship between mesh vertices and skeleton joints. The proposed method exploits the benefits of the novel Multi-Aggregator Graph Convolution that combines the results of different aggregators during the summarizing step of the Message-Passing scheme, helping the operation to generalize for unseen topologies. Experimental results demonstrate the effectiveness of the contributions of our novel architecture, with SkinningNet outperforming current state-of-the-art alternatives.

https://imatge-upc.github.io/skinningnet/

}, url = {https://imatge-upc.github.io/skinningnet/}, author = {Mosella-Montoro, Albert and Ruiz-Hidalgo, J.} } @article {aMosella-Montoro21, title = {2D{\textendash}3D Geometric Fusion network using Multi-Neighbourhood Graph Convolution for RGB-D indoor scene classification}, journal = {Information Fusion}, volume = {76}, year = {2021}, month = {12/2021}, chapter = {46-54}, abstract = {

Multi-modal fusion has been proved to help enhance the performance of scene classification tasks. This paper presents a 2D-3D Fusion stage that combines 3D Geometric Features with 2D Texture Features obtained by 2D Convolutional Neural Networks. To get a robust 3D Geometric embedding, a network that uses two novel layers is proposed. The first layer, Multi-Neighbourhood Graph Convolution, aims to learn a more robust geometric descriptor of the scene combining two different neighbourhoods: one in the Euclidean space and the other in the Feature space. The second proposed layer, Nearest Voxel Pooling, improves the performance of the well-known Voxel Pooling. Experimental results, using NYU-Depth-V2 and SUN RGB-D datasets, show that the proposed method outperforms the current state-of-the-art in RGB-D indoor scene classification task.\ 

https://imatge-upc.github.io/munegc/

}, doi = {10.1016/j.inffus.2021.05.002}, url = {https://imatge-upc.github.io/munegc/}, author = {Mosella-Montoro, Albert and Ruiz-Hidalgo, J.} } @conference {cRuiz-Hidalgo19, title = {Residual Attention Graph Convolutional Network for Geometric 3D Scene Classification}, booktitle = {IEEE Conference on Computer Vision Workshop (ICCVW)}, year = {2019}, month = {11/2019}, publisher = {IEEE}, organization = {IEEE}, address = {Seoul, Korea}, abstract = {

Geometric 3D scene classification is a very challenging task. Current methodologies extract the geometric information using only a depth channel provided by an RGBD sensor. These kinds of methodologies introduce possible errors due to missing local geometric context in the depth channel. This work proposes a novel Residual Attention Graph Convolutional Network that exploits the intrinsic geometric context inside a 3D space without using any kind of point features, allowing the use of organized or unorganized 3D data. Experiments are done in NYU Depth v1 and SUN-RGBD datasets to study the different configurations and to demonstrate the effectiveness of the proposed method. Experimental results show that the proposed method outperforms current state-of-the-art in geometric 3D scene classification tasks.\ 

}, doi = {10.1109/ICCVW.2019.00507}, url = {https://imatge-upc.github.io/ragc/}, author = {Mosella-Montoro, Albert and Ruiz-Hidalgo, J.} }