@conference {cde-Mas-Gimenez25a, title = {Multimodal data acquisition prototype for autonomous driving in adverse weather conditions}, booktitle = {XIV Reuni{\'o}n OptoElectr{\'o}nica (OPTOEL 2025)}, year = {2025}, month = {07/2025}, publisher = {OPTOEL}, organization = {OPTOEL}, address = {Terrassa, Spain}, abstract = {

Autonomous driving has made significant progress in recent years, but adverse weather conditions still remain a major challenge for perception and decision-making algorithms. This work presents a multimodal data acquisition prototype designed to enhance autonomous vehicle perception in challenging environments such as fog, heavy rain, and snow. The system is mounted on a Dacia Duster and features a diverse sensor suite including visible, Short-Wave Infra-Red, thermal, and polarimetric cameras with a solid-state LiDAR and automotive radars. The whole system is calibrated with dedicated calibration boards developed on purpose to enable data fusion across all modalities. Initial results demonstrate the effectiveness of our approach, showing that critical environmental features can be detected reliably across various weather conditions. Future work includes the release of a fully annotated multimodal dataset to support further research in adverse weather perception.

}, isbn = {979-13-87613-54-9}, doi = {10.5821/ebook-9791387613549}, url = {https://optoel.es/2025/wp-content/uploads/2025/07/LIBRO-ACTAS-OPTOEL2025.pdf$\#$?page=403}, author = {Gerard de-Mas-Gim{\'e}nez and Adri{\`a} Subirana and Pablo Garc{\'\i}a-G{\'o}mez and Eduardo Bernal P{\'e}rez and Jordi Riu and Josep R Casas and Santiago Royo} } @conference {cde-Mas-Gimenez25, title = {Multimodal sensing prototype for robust autonomous driving under adverse weather conditions}, booktitle = {SPIE Optical Metrology: Optical Measurement Systems for Industrial Inspection XIV}, year = {2025}, month = {06/2025}, publisher = {SPIE}, organization = {SPIE}, address = {Munich, Germany}, abstract = {

Current autonomous driving datasets face significant limitations in adverse weather conditions diversity and sensor generalization. Additionally, commonly used sensors like visible cameras or rotating LiDARs, struggle to perform under harsh weather conditions. To address these challenges, this work introduces a multimodal data acquisition system that integrates high-resolution solid-state LiDAR, automotive RADARs, a combination of visible, thermal, SWIR, and polarimetric cameras, and a GNSS/INS system for odometry and localization. This diverse sensor suite ensures robust performance in low-visibility environments, such as fog or heavy rain, by providing complementary and redundant information. Controlled by an autonomous-safe Nvidia DRIVE AGX with a ROS-based architecture, the system enables precise spatial calibration, temporal synchronization, and realtime data fusion and perception algorithms across an overlapped field of view of 60{\textopenbullet}x 20{\textopenbullet}. With this system, this work aims to publish in the coming months an open-source multimodal labeled dataset for autonomous driving complemented by synthetic data generation through a digital twin in Nvidia{\textquoteright}s Omniverse platform. The data set will have a similar structure as the well-known nuScenes, and will have a similar developer kit to navigate through it with ease. The data set will support a wide range of autonomous driving applications like 3D multimodal object detection and tracking, SLAM, depth completion, and perception enhancement through challenging scenarios.

}, doi = {10.1117/12.3062388}, author = {Gerard de-Mas-Gim{\'e}nez and Adri{\`a} Subirana and Pablo Garc{\'\i}a-G{\'o}mez and Eduardo Bernal P{\'e}rez and Josep R Casas and Santiago Royo} } @conference {cde-Mas-Gimenez25b, title = {Preview of a multimodal data set for robust and safe autonomous driving under adverse weather conditions}, booktitle = {IMEKO PhotoMet - International Symposium on Modern Photonic Metrology}, year = {2025}, month = {09/2025}, publisher = {11 IMEKO PhotoMet}, organization = {11 IMEKO PhotoMet}, address = {Modena, Italy}, abstract = {

Autonomous driving has made remarkable strides in recent years, yet adverse weather conditions such as fog, heavy rain, snow, and low-light environments remain major obstacles for perception systems. These conditions reduce visibility and introduce complex artifacts like scattering, reflection, and sensor saturation, which degrade the performance of traditional camera-LiDAR-based perception algorithms. Current state-of-theart autonomous driving data sets avoid these edge cases. To address this, we introduce a new multimodal data set specifically curated to support research in robust perception under challenging environmental conditions. The data set was collected using a multimodal platform installed on a custom prototype which includes a diverse suite of sensors: visible RGB cameras, a Short-Wave Infra-Red (SWIR) camera, long-wave infrared thermal camera, and polarimetric camera. In addition to these optical sensors, the system incorporates a high-resolution solid-state LiDAR and two automotive-grade radars. These modalities were selected for their complementary responses to different environmental challenges. The prototype has a [60{\textordmasculine},40{\textordmasculine}] of overlapped FOV with all modalities ensuring enough redundancy to provide a good understanding of the vehicle{\textquoteright}s surroundings. Additionally, a GNSS/INS system is mounted to provide with localization and vehicle odometry. All sensors are temporally paired using a best effort synchronization algorithm. They are also spatially registered using multimodal calibration boards, enabling accurate data fusion even at long distances. The data set covers a wide range of real-world driving scenarios with urban, road and highway at high-velocities scenes across various lighting and weather conditions. From clear sky, to heavy rain, snow and even fog. Each frame includes 2D and 3D annotations for object detection tasks. The data set is structured as a relational database with an architecture similar to the reference data set, nuScenes. To ease visualization and navigation and through the data set, an extended python developer kit tool will be published. This work offers a preview of a multimodal data set which offers a rich and unique resource to the research community, enabling the development and evaluation of algorithms designed to operate reliably in degraded visibility. The data set is expected to be open-sourced later this year.

}, keywords = {Adverse weather conditions, autonomous driving, Data fusion, LiDAR, Multimodal imaging}, url = {https://photomet.org/abstracts}, author = {Gerard de-Mas-Gim{\'e}nez and Adri{\`a} Subirana and Eduardo Bernal P{\'e}rez and Jordi Riu and Josep R Casas and Pablo Garc{\'\i}a-G{\'o}mez and S. Royo} } @article {aPlasencia23, title = {A Preliminary Study of Deep Learning Sensor Fusion for Pedestrian Detection}, journal = {Sensors}, volume = {23}, year = {2023}, month = {04/2023}, chapter = {4167}, abstract = {

Most pedestrian detection methods focus on bounding boxes based on fusing RGB with lidar. These methods do not relate to how the human eye perceives objects in the real world. Furthermore, lidar and vision can have difficulty detecting pedestrians in scattered environments, and radar can be used to overcome this problem. Therefore, the motivation of this work is to explore, as a preliminary step, the feasibility of fusing lidar, radar, and RGB for pedestrian detection that potentially can be used for autonomous driving that uses a fully connected convolutional neural network architecture for multimodal sensors. The core of the network is based on SegNet, a pixel-wise semantic segmentation network. In this context, lidar and radar were incorporated by transforming them from 3D pointclouds into 2D gray images with 16-bit depths, and RGB images were incorporated with three channels. The proposed architecture uses a single SegNet for each sensor reading, and the outputs are then applied to a fully connected neural network to fuse the three modalities of sensors. Afterwards, an up-sampling network is applied to recover the fused data. Additionally, a custom dataset of 60 images was proposed for training the architecture, with an additional 10 for evaluation and 10 for testing, giving a total of 80 images. The experiment results show a training mean pixel accuracy of 99.7\% and a training mean intersection over union of 99.5\%. Also, the testing mean of the IoU was 94.4\%, and the testing pixel accuracy was 96.2\%. These metric results have successfully demonstrated the effectiveness of using semantic segmentation for pedestrian detection under the modalities of three sensors. Despite some overfitting in the model during experimentation, it performed well in detecting people in test mode. Therefore, it is worth emphasizing that the focus of this work is to show that this method is feasible to be used, as it works regardless of the size of the dataset. Also, a bigger dataset would be necessary to achieve a more appropiate training. This method gives the advantage of detecting pedestrians as the human eye does, thereby resulting in less ambiguity. Additionally, this work has also proposed an extrinsic calibration matrix method for sensor alignment between radar and lidar based on singular value decomposition.

}, keywords = {autonomous driving, Convolutional Neural Networks, sensor calibration, sensor fusion}, doi = {10.3390/s23084167}, url = {https://www.mdpi.com/1424-8220/23/8/4167}, author = {Alfredo Ch{\'a}vez Plasencia and Pablo Garc{\'\i}a-G{\'o}mez and Eduardo Bernal P{\'e}rez and Gerard de-Mas-Gim{\'e}nez and Josep R Casas and S. Royo} }