@conference {cde-Mas-Gimenez25a, title = {Multimodal data acquisition prototype for autonomous driving in adverse weather conditions}, booktitle = {XIV Reuni{\'o}n OptoElectr{\'o}nica (OPTOEL 2025)}, year = {2025}, month = {07/2025}, publisher = {OPTOEL}, organization = {OPTOEL}, address = {Terrassa, Spain}, abstract = {

Autonomous driving has made significant progress in recent years, but adverse weather conditions still remain a major challenge for perception and decision-making algorithms. This work presents a multimodal data acquisition prototype designed to enhance autonomous vehicle perception in challenging environments such as fog, heavy rain, and snow. The system is mounted on a Dacia Duster and features a diverse sensor suite including visible, Short-Wave Infra-Red, thermal, and polarimetric cameras with a solid-state LiDAR and automotive radars. The whole system is calibrated with dedicated calibration boards developed on purpose to enable data fusion across all modalities. Initial results demonstrate the effectiveness of our approach, showing that critical environmental features can be detected reliably across various weather conditions. Future work includes the release of a fully annotated multimodal dataset to support further research in adverse weather perception.

}, isbn = {979-13-87613-54-9}, doi = {10.5821/ebook-9791387613549}, url = {https://optoel.es/2025/wp-content/uploads/2025/07/LIBRO-ACTAS-OPTOEL2025.pdf$\#$?page=403}, author = {Gerard de-Mas-Gim{\'e}nez and Adri{\`a} Subirana and Pablo Garc{\'\i}a-G{\'o}mez and Eduardo Bernal P{\'e}rez and Jordi Riu and Casas, J. and Santiago Royo} } @conference {cde-Mas-Gimenez25, title = {Multimodal sensing prototype for robust autonomous driving under adverse weather conditions}, booktitle = {SPIE Optical Metrology: Optical Measurement Systems for Industrial Inspection XIV}, year = {2025}, month = {06/2025}, publisher = {SPIE}, organization = {SPIE}, address = {Munich, Germany}, abstract = {

Current autonomous driving datasets face significant limitations in adverse weather conditions diversity and sensor generalization. Additionally, commonly used sensors like visible cameras or rotating LiDARs, struggle to perform under harsh weather conditions. To address these challenges, this work introduces a multimodal data acquisition system that integrates high-resolution solid-state LiDAR, automotive RADARs, a combination of visible, thermal, SWIR, and polarimetric cameras, and a GNSS/INS system for odometry and localization. This diverse sensor suite ensures robust performance in low-visibility environments, such as fog or heavy rain, by providing complementary and redundant information. Controlled by an autonomous-safe Nvidia DRIVE AGX with a ROS-based architecture, the system enables precise spatial calibration, temporal synchronization, and realtime data fusion and perception algorithms across an overlapped field of view of 60{\textopenbullet}x 20{\textopenbullet}. With this system, this work aims to publish in the coming months an open-source multimodal labeled dataset for autonomous driving complemented by synthetic data generation through a digital twin in Nvidia{\textquoteright}s Omniverse platform. The data set will have a similar structure as the well-known nuScenes, and will have a similar developer kit to navigate through it with ease. The data set will support a wide range of autonomous driving applications like 3D multimodal object detection and tracking, SLAM, depth completion, and perception enhancement through challenging scenarios.

}, doi = {10.1117/12.3062388}, author = {Gerard de-Mas-Gim{\'e}nez and Adri{\`a} Subirana and Pablo Garc{\'\i}a-G{\'o}mez and Eduardo Bernal P{\'e}rez and Casas, J. and Santiago Royo} } @article {ade-Mas-Gimenez23, title = {Gradient-Based Metrics for the Evaluation of Image Defogging}, journal = {World Electric Vehicle Journal}, volume = {14}, year = {2023}, month = {09/2023}, chapter = {254}, abstract = {

Fog, haze, or smoke are standard atmospheric phenomena that dramatically compromise the overall visibility of any scene, critically affecting features such as the illumination, contrast, and contour detection of objects. The decrease in visibility compromises the performance of computer vision algorithms such as pattern recognition and segmentation, some of which are very relevant to decision-making in the field of autonomous vehicles. Several dehazing methods have been proposed that either need to estimate fog parameters through physical models or are statistically based. But physical parameters greatly depend on the scene conditions, and statistically based methods require large datasets of natural foggy images together with the original images without fog, i.e., the ground truth, for evaluation. Obtaining proper fog-less ground truth images for pixel-to-pixel evaluation is costly and time-consuming, and this fact hinders progress in the field. This paper aims to tackle this issue by proposing gradient-based metrics for image defogging evaluation that do not require a ground truth image without fog or a physical model. A comparison of the proposed metrics with metrics already used in the NTIRE 2018 defogging challenge as well as several state-of-the-art defogging evaluation metrics is performed to prove its effectiveness in a general situation, showing comparable results to conventional metrics and an improvement in the no-reference scene. A Matlab implementation of the proposed metrics has been developed and it is open-sourced in a public GitHub repository.

}, doi = {10.3390/wevj14090254}, url = {https://www.mdpi.com/2032-6653/14/9/254}, author = {Gerard de-Mas-Gim{\'e}nez and Pablo Garc{\'\i}a-G{\'o}mez and Casas, J. and S. Royo} } @article {aPlasencia23, title = {A Preliminary Study of Deep Learning Sensor Fusion for Pedestrian Detection}, journal = {Sensors}, volume = {23}, year = {2023}, month = {04/2023}, chapter = {4167}, abstract = {

Most pedestrian detection methods focus on bounding boxes based on fusing RGB with lidar. These methods do not relate to how the human eye perceives objects in the real world. Furthermore, lidar and vision can have difficulty detecting pedestrians in scattered environments, and radar can be used to overcome this problem. Therefore, the motivation of this work is to explore, as a preliminary step, the feasibility of fusing lidar, radar, and RGB for pedestrian detection that potentially can be used for autonomous driving that uses a fully connected convolutional neural network architecture for multimodal sensors. The core of the network is based on SegNet, a pixel-wise semantic segmentation network. In this context, lidar and radar were incorporated by transforming them from 3D pointclouds into 2D gray images with 16-bit depths, and RGB images were incorporated with three channels. The proposed architecture uses a single SegNet for each sensor reading, and the outputs are then applied to a fully connected neural network to fuse the three modalities of sensors. Afterwards, an up-sampling network is applied to recover the fused data. Additionally, a custom dataset of 60 images was proposed for training the architecture, with an additional 10 for evaluation and 10 for testing, giving a total of 80 images. The experiment results show a training mean pixel accuracy of 99.7\% and a training mean intersection over union of 99.5\%. Also, the testing mean of the IoU was 94.4\%, and the testing pixel accuracy was 96.2\%. These metric results have successfully demonstrated the effectiveness of using semantic segmentation for pedestrian detection under the modalities of three sensors. Despite some overfitting in the model during experimentation, it performed well in detecting people in test mode. Therefore, it is worth emphasizing that the focus of this work is to show that this method is feasible to be used, as it works regardless of the size of the dataset. Also, a bigger dataset would be necessary to achieve a more appropiate training. This method gives the advantage of detecting pedestrians as the human eye does, thereby resulting in less ambiguity. Additionally, this work has also proposed an extrinsic calibration matrix method for sensor alignment between radar and lidar based on singular value decomposition.

}, keywords = {autonomous driving, Convolutional Neural Networks, sensor calibration, sensor fusion}, doi = {10.3390/s23084167}, url = {https://www.mdpi.com/1424-8220/23/8/4167}, author = {Alfredo Ch{\'a}vez Plasencia and Pablo Garc{\'\i}a-G{\'o}mez and Eduardo Bernal P{\'e}rez and Gerard de-Mas-Gim{\'e}nez and Casas, J. and S. Royo} } @conference {cde-Mas-Gimenez23, title = {Single image defogging and evaluation with deep neural networks without the need of ground truth}, booktitle = {SPIE Automated Visual Inspection and Machine Vision V}, year = {2023}, month = {05/2023}, publisher = {Proc. SPIE 12623}, organization = {Proc. SPIE 12623}, address = {Munich}, abstract = {

Fog dramatically compromises the overall visibility of any scene, critically affecting features such as objects{\textquoteright} illumination, contrast, and contours. The decrease in visibility compromises the performance of Computer Vision algorithms such as pattern recognition and segmentation, some of them very relevant to decision-making in the rise of autonomous-driven vehicles. Many dehazing methods have been proposed. However, to the best of our knowledge, all currently used metrics do compare the defogged image to its ground truth, usually the same scene on a non-foggy day, or estimate physical parameters from the scene. This hinders progress in the field, as obtaining proper ground truth images is not always possible and becomes costly and time-consuming because physical parameters greatly depend on the scene conditions. This work aims to tackle this issue by proposing a real-time operating defogging network that only takes an RGB image of the fogged scene as input, performs the defogging, and uses a contour-based metric for Single Image Defogging evaluation even when the ground truth is not available, which is the most common situation. The proposed metric only requires the original hazy image and the image after the defogging procedure. We trained our network using a novel two-stage pipeline with the DENSE dataset and compared our method and metric with currently used metrics and other defogging techniques with the NTIRE 2018 defogging challenge to prove their effectiveness.

}, doi = {10.1117/12.2673442}, author = {Gerard de-Mas-Gim{\'e}nez and Pablo Garc{\'\i}a-G{\'o}mez and Casas, J. and S. Royo} }