From e08cc180d218ca6325a95fdc2b309812e396f3fe Mon Sep 17 00:00:00 2001 From: Robert Haschke Date: Fri, 12 Jun 2020 22:08:58 +0200 Subject: [PATCH 1/2] Fix memory access in case of 3-byte pixel formats --- src/rviz/selection/selection_manager.cpp | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/rviz/selection/selection_manager.cpp b/src/rviz/selection/selection_manager.cpp index 55754df2a9..582dcf215b 100644 --- a/src/rviz/selection/selection_manager.cpp +++ b/src/rviz/selection/selection_manager.cpp @@ -582,17 +582,20 @@ void SelectionManager::unpackColors(const Ogre::PixelBox& box, V_CollObject& pix pixels.clear(); pixels.reserve(w * h); + size_t size = Ogre::PixelUtil::getMemorySize(1, 1, 1, box.format); for (int y = 0; y < h; y++) { for (int x = 0; x < w; x++) { - uint32_t pos = (x + y * w) * 4; - - uint32_t pix_val = *(uint32_t*)((uint8_t*)box.data + pos); - uint32_t handle = colorToHandle(box.format, pix_val); - - pixels.push_back(handle); + if (size == 4) // In case of a 4-byte color format, we can directly process the 32-bit values + { + uint32_t pos = (x + y * w) * 4; + uint32_t pix_val = *(uint32_t*)((uint8_t*)box.data + pos); + pixels.push_back(colorToHandle(box.format, pix_val)); + } + else + pixels.push_back(colorToHandle(const_cast(box).getColourAt(x, y, 1))); } } } From 6b254a67282a05b8cf05b320ee80a58f224b2364 Mon Sep 17 00:00:00 2001 From: Robert Haschke Date: Sat, 13 Jun 2020 07:40:58 +0200 Subject: [PATCH 2/2] change private API instead of const-casting --- src/rviz/selection/selection_manager.cpp | 6 +++--- src/rviz/selection/selection_manager.h | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/rviz/selection/selection_manager.cpp b/src/rviz/selection/selection_manager.cpp index 582dcf215b..954ceded8c 100644 --- a/src/rviz/selection/selection_manager.cpp +++ b/src/rviz/selection/selection_manager.cpp @@ -575,7 +575,7 @@ void SelectionManager::setHighlightRect(Ogre::Viewport* viewport, int x1, int y1 highlight_rectangle_->setCorners(nx1, ny1, nx2, ny2); } -void SelectionManager::unpackColors(const Ogre::PixelBox& box, V_CollObject& pixels) +void SelectionManager::unpackColors(Ogre::PixelBox& box, V_CollObject& pixels) { int w = box.getWidth(); int h = box.getHeight(); @@ -594,8 +594,8 @@ void SelectionManager::unpackColors(const Ogre::PixelBox& box, V_CollObject& pix uint32_t pix_val = *(uint32_t*)((uint8_t*)box.data + pos); pixels.push_back(colorToHandle(box.format, pix_val)); } - else - pixels.push_back(colorToHandle(const_cast(box).getColourAt(x, y, 1))); + else // otherwise perform "official" transformation into float-based Ogre::ColourValue and back + pixels.push_back(colorToHandle(box.getColourAt(x, y, 1))); } } } diff --git a/src/rviz/selection/selection_manager.h b/src/rviz/selection/selection_manager.h index cf20afc968..8b481a3f3a 100644 --- a/src/rviz/selection/selection_manager.h +++ b/src/rviz/selection/selection_manager.h @@ -275,7 +275,7 @@ private Q_SLOTS: unsigned texture_width, unsigned textured_height); - void unpackColors(const Ogre::PixelBox& box, V_CollObject& pixels); + void unpackColors(Ogre::PixelBox& box, V_CollObject& pixels); void setDepthTextureSize(unsigned width, unsigned height);