From e5ef587b50079e71cce929e4f2d3b380c030b0d7 Mon Sep 17 00:00:00 2001 From: Robert Haschke Date: Tue, 23 Jun 2020 16:54:57 +0200 Subject: [PATCH] Fix memory access in case of 3-byte pixel formats (#1519) --- src/rviz/selection/selection_manager.cpp | 17 ++++++++++------- src/rviz/selection/selection_manager.h | 2 +- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/src/rviz/selection/selection_manager.cpp b/src/rviz/selection/selection_manager.cpp index 55754df2a9..954ceded8c 100644 --- a/src/rviz/selection/selection_manager.cpp +++ b/src/rviz/selection/selection_manager.cpp @@ -575,24 +575,27 @@ void SelectionManager::setHighlightRect(Ogre::Viewport* viewport, int x1, int y1 highlight_rectangle_->setCorners(nx1, ny1, nx2, ny2); } -void SelectionManager::unpackColors(const Ogre::PixelBox& box, V_CollObject& pixels) +void SelectionManager::unpackColors(Ogre::PixelBox& box, V_CollObject& pixels) { int w = box.getWidth(); int h = box.getHeight(); pixels.clear(); pixels.reserve(w * h); + size_t size = Ogre::PixelUtil::getMemorySize(1, 1, 1, box.format); for (int y = 0; y < h; y++) { for (int x = 0; x < w; x++) { - uint32_t pos = (x + y * w) * 4; - - uint32_t pix_val = *(uint32_t*)((uint8_t*)box.data + pos); - uint32_t handle = colorToHandle(box.format, pix_val); - - pixels.push_back(handle); + if (size == 4) // In case of a 4-byte color format, we can directly process the 32-bit values + { + uint32_t pos = (x + y * w) * 4; + uint32_t pix_val = *(uint32_t*)((uint8_t*)box.data + pos); + pixels.push_back(colorToHandle(box.format, pix_val)); + } + else // otherwise perform "official" transformation into float-based Ogre::ColourValue and back + pixels.push_back(colorToHandle(box.getColourAt(x, y, 1))); } } } diff --git a/src/rviz/selection/selection_manager.h b/src/rviz/selection/selection_manager.h index cf20afc968..8b481a3f3a 100644 --- a/src/rviz/selection/selection_manager.h +++ b/src/rviz/selection/selection_manager.h @@ -275,7 +275,7 @@ private Q_SLOTS: unsigned texture_width, unsigned textured_height); - void unpackColors(const Ogre::PixelBox& box, V_CollObject& pixels); + void unpackColors(Ogre::PixelBox& box, V_CollObject& pixels); void setDepthTextureSize(unsigned width, unsigned height);