From 566f7f830b72a5443bf3045a34c42d1716b4ef38 Mon Sep 17 00:00:00 2001 From: Fabien Servant Date: Wed, 30 Sep 2020 10:52:58 +0200 Subject: [PATCH 01/79] extract modules from warping --- src/aliceVision/CMakeLists.txt | 3 +- src/aliceVision/panorama/CMakeLists.txt | 24 ++ src/aliceVision/panorama/boundingBox.hpp | 21 + src/aliceVision/panorama/coordinatesMap.cpp | 94 +++++ src/aliceVision/panorama/coordinatesMap.hpp | 47 +++ src/aliceVision/panorama/distance.cpp | 45 ++ src/aliceVision/panorama/distance.hpp | 11 + src/aliceVision/panorama/gaussian.cpp | 83 ++++ src/aliceVision/panorama/gaussian.hpp | 36 ++ src/aliceVision/panorama/remapBbox.cpp | 393 ++++++++++++++++++ src/aliceVision/panorama/remapBbox.hpp | 13 + src/aliceVision/panorama/sphericalMapping.cpp | 46 ++ src/aliceVision/panorama/sphericalMapping.hpp | 29 ++ src/aliceVision/panorama/warper.cpp | 116 ++++++ src/aliceVision/panorama/warper.hpp | 43 ++ src/software/pipeline/CMakeLists.txt | 1 + 16 files changed, 1004 insertions(+), 1 deletion(-) create mode 100644 src/aliceVision/panorama/CMakeLists.txt create mode 100644 src/aliceVision/panorama/boundingBox.hpp create mode 100644 src/aliceVision/panorama/coordinatesMap.cpp create mode 100644 src/aliceVision/panorama/coordinatesMap.hpp create mode 100644 src/aliceVision/panorama/distance.cpp create mode 100644 src/aliceVision/panorama/distance.hpp create mode 100644 src/aliceVision/panorama/gaussian.cpp create mode 100644 src/aliceVision/panorama/gaussian.hpp create mode 100644 src/aliceVision/panorama/remapBbox.cpp create mode 100644 src/aliceVision/panorama/remapBbox.hpp create mode 100644 src/aliceVision/panorama/sphericalMapping.cpp create mode 100644 src/aliceVision/panorama/sphericalMapping.hpp create mode 100644 src/aliceVision/panorama/warper.cpp create mode 100644 src/aliceVision/panorama/warper.hpp diff --git a/src/aliceVision/CMakeLists.txt b/src/aliceVision/CMakeLists.txt index cfd57ea0f5..342c5fd633 100644 --- a/src/aliceVision/CMakeLists.txt +++ b/src/aliceVision/CMakeLists.txt @@ -5,6 +5,7 @@ add_subdirectory(numeric) add_subdirectory(system) add_subdirectory(stl) add_subdirectory(utils) +add_subdirectory(panorama) # SfM modules @@ -37,7 +38,7 @@ if(ALICEVISION_BUILD_SFM) endif() if(ALICEVISION_BUILD_HDR) -add_subdirectory(hdr) + add_subdirectory(hdr) endif() # MVS modules diff --git a/src/aliceVision/panorama/CMakeLists.txt b/src/aliceVision/panorama/CMakeLists.txt new file mode 100644 index 0000000000..915fdc9391 --- /dev/null +++ b/src/aliceVision/panorama/CMakeLists.txt @@ -0,0 +1,24 @@ +# Headers +set(panorama_files_headers + boundingBox.hpp + warper.hpp +) + +# Sources +set(panorama_files_sources + warper.cpp + gaussian.cpp + coordinatesMap.cpp + distance.cpp + remapBbox.cpp + sphericalMapping.cpp +) + +alicevision_add_library(aliceVision_panorama + SOURCES ${panorama_files_headers} ${panorama_files_sources} + PUBLIC_LINKS + aliceVision_numeric + PRIVATE_LINKS + aliceVision_system + aliceVision_image +) diff --git a/src/aliceVision/panorama/boundingBox.hpp b/src/aliceVision/panorama/boundingBox.hpp new file mode 100644 index 0000000000..920efa0952 --- /dev/null +++ b/src/aliceVision/panorama/boundingBox.hpp @@ -0,0 +1,21 @@ +#pragma once + + +struct BoundingBox { + int left; + int top; + int width; + int height; + + BoundingBox() = default; + + BoundingBox(int l, int t, int w, int h) : left(l), top(t), width(w), height(h) {} + + int getRight() const { + return left + width; + } + + int getBottom() const { + return top + height; + } +}; \ No newline at end of file diff --git a/src/aliceVision/panorama/coordinatesMap.cpp b/src/aliceVision/panorama/coordinatesMap.cpp new file mode 100644 index 0000000000..aac3380438 --- /dev/null +++ b/src/aliceVision/panorama/coordinatesMap.cpp @@ -0,0 +1,94 @@ +#pragma once + +#include "coordinatesMap.hpp" + +#include "sphericalMapping.hpp" + +namespace aliceVision { + + +bool CoordinatesMap::build(const std::pair & panoramaSize, const geometry::Pose3 & pose, const aliceVision::camera::IntrinsicBase & intrinsics, const BoundingBox &coarseBbox) { + + /* Effectively compute the warping map */ + _coordinates = aliceVision::image::Image(coarseBbox.width, coarseBbox.height, false); + _mask = aliceVision::image::Image(coarseBbox.width, coarseBbox.height, true, 0); + + + #pragma omp parallel for + for (size_t y = 0; y < coarseBbox.height; y++) { + + size_t cy = y + coarseBbox.top; + + + for (size_t x = 0; x < coarseBbox.width; x++) { + + size_t cx = x + coarseBbox.left; + + Vec3 ray = SphericalMapping::fromEquirectangular(Vec2(cx, cy), panoramaSize.first, panoramaSize.second); + + /** + * Check that this ray should be visible. + * This test is camera type dependent + */ + Vec3 transformedRay = pose(ray); + if (!intrinsics.isVisibleRay(transformedRay)) { + continue; + } + + /** + * Project this ray to camera pixel coordinates + */ + const Vec2 pix_disto = intrinsics.project(pose, ray, true); + + /** + * Ignore invalid coordinates + */ + if (!intrinsics.isVisible(pix_disto)) { + continue; + } + + _coordinates(y, x) = pix_disto; + _mask(y, x) = 1; + } + } + + _offset_x = coarseBbox.left; + _offset_y = coarseBbox.top; + + return true; +} + +bool CoordinatesMap::computeScale(double & result, float ratioUpscale) { + + std::vector scales; + size_t real_height = _coordinates.Height(); + size_t real_width = _coordinates.Width(); + + for (int i = 1; i < real_height - 2; i++) { + for (int j = 1; j < real_width - 2; j++) { + if (!_mask(i, j) || !_mask(i, j + 1) || !_mask(i + 1, j)) { + continue; + } + + double dxx = _coordinates(i, j + 1).x() - _coordinates(i, j).x(); + double dxy = _coordinates(i + 1, j).x() - _coordinates(i, j).x(); + double dyx = _coordinates(i, j + 1).y() - _coordinates(i, j).y(); + double dyy = _coordinates(i + 1, j).y() - _coordinates(i, j).y(); + + double det = std::abs(dxx*dyy - dxy*dyx); + if (det < 1e-12) continue; + + scales.push_back(det); + } + } + + if (scales.empty()) return false; + + std::sort(scales.begin(), scales.end()); + int selected_index = int(floor(float(scales.size() - 1) * ratioUpscale)); + result = sqrt(scales[selected_index]); + + return true; +} + +} \ No newline at end of file diff --git a/src/aliceVision/panorama/coordinatesMap.hpp b/src/aliceVision/panorama/coordinatesMap.hpp new file mode 100644 index 0000000000..d07cc3020d --- /dev/null +++ b/src/aliceVision/panorama/coordinatesMap.hpp @@ -0,0 +1,47 @@ +#pragma once + + +#include +#include + +#include "boundingBox.hpp" + +namespace aliceVision { + +class CoordinatesMap { +public: + /** + * Build coordinates map given camera properties + * @param panoramaSize desired output panoramaSize + * @param pose the camera pose wrt an arbitrary reference frame + * @param intrinsics the camera intrinsics + */ + bool build(const std::pair & panoramaSize, const geometry::Pose3 & pose, const aliceVision::camera::IntrinsicBase & intrinsics, const BoundingBox &coarseBbox); + + bool computeScale(double & result, float ratioUpscale); + + size_t getOffsetX() const { + return _offset_x; + } + + size_t getOffsetY() const { + return _offset_y; + } + + const aliceVision::image::Image & getCoordinates() const { + return _coordinates; + } + + const aliceVision::image::Image & getMask() const { + return _mask; + } + +private: + size_t _offset_x = 0; + size_t _offset_y = 0; + + aliceVision::image::Image _coordinates; + aliceVision::image::Image _mask; +}; + +} \ No newline at end of file diff --git a/src/aliceVision/panorama/distance.cpp b/src/aliceVision/panorama/distance.cpp new file mode 100644 index 0000000000..88b54b90e5 --- /dev/null +++ b/src/aliceVision/panorama/distance.cpp @@ -0,0 +1,45 @@ +#pragma once + +#include "distance.hpp" + +namespace aliceVision { + +bool distanceToCenter(aliceVision::image::Image & _weights, const CoordinatesMap & map) { + + const aliceVision::image::Image & coordinates = map.getCoordinates(); + const aliceVision::image::Image & mask = map.getMask(); + + float w = static_cast(coordinates.Width()); + float h = static_cast(coordinates.Height()); + + float cx = w / 2.0f; + float cy = h / 2.0f; + + _weights = aliceVision::image::Image(coordinates.Width(), coordinates.Height()); + + for (int i = 0; i < _weights.Height(); i++) { + for (int j = 0; j < _weights.Width(); j++) { + + _weights(i, j) = 0.0f; + + bool valid = mask(i, j); + if (!valid) { + continue; + } + + const Vec2 & coords = coordinates(i, j); + + float x = coords(0); + float y = coords(1); + + float wx = 1.0f - std::abs((x - cx) / cx); + float wy = 1.0f - std::abs((y - cy) / cy); + + _weights(i, j) = wx * wy; + } + } + + return true; +} + +} \ No newline at end of file diff --git a/src/aliceVision/panorama/distance.hpp b/src/aliceVision/panorama/distance.hpp new file mode 100644 index 0000000000..781607ca9d --- /dev/null +++ b/src/aliceVision/panorama/distance.hpp @@ -0,0 +1,11 @@ +#pragma once + +#include + +#include "coordinatesMap.hpp" + +namespace aliceVision { + +bool distanceToCenter(aliceVision::image::Image & _weights, const CoordinatesMap & map); + +} \ No newline at end of file diff --git a/src/aliceVision/panorama/gaussian.cpp b/src/aliceVision/panorama/gaussian.cpp new file mode 100644 index 0000000000..e009a1ec95 --- /dev/null +++ b/src/aliceVision/panorama/gaussian.cpp @@ -0,0 +1,83 @@ +#include "gaussian.hpp" + +#include + +namespace aliceVision { + +GaussianPyramidNoMask::GaussianPyramidNoMask(const size_t width_base, const size_t height_base, const size_t limit_scales) : + _width_base(width_base), + _height_base(height_base) +{ + /** + * Compute optimal scale + * The smallest level will be at least of size min_size + */ + size_t min_dim = std::min(_width_base, _height_base); + size_t min_size = 32; + _scales = std::min(limit_scales, static_cast(floor(log2(double(min_dim) / float(min_size))))); + + + /** + * Create pyramid + **/ + size_t new_width = _width_base; + size_t new_height = _height_base; + for (int i = 0; i < _scales; i++) { + + _pyramid_color.push_back(image::Image(new_width, new_height, true, image::RGBfColor(0))); + _filter_buffer.push_back(image::Image(new_width, new_height, true, image::RGBfColor(0))); + new_height /= 2; + new_width /= 2; + } +} + +bool GaussianPyramidNoMask::process(const image::Image & input) { + + if (input.Height() != _pyramid_color[0].Height()) return false; + if (input.Width() != _pyramid_color[0].Width()) return false; + + + /** + * Kernel + */ + oiio::ImageBuf K; + oiio::ImageBufAlgo::make_kernel(K, "gaussian", 5, 5); + + /** + * Build pyramid + */ + _pyramid_color[0] = input; + for (int lvl = 0; lvl < _scales - 1; lvl++) { + + const image::Image & source = _pyramid_color[lvl]; + image::Image & dst = _filter_buffer[lvl]; + + oiio::ImageSpec spec(source.Width(), source.Height(), 3, oiio::TypeDesc::FLOAT); + + const oiio::ImageBuf inBuf(spec, const_cast(source.data())); + oiio::ImageBuf outBuf(spec, dst.data()); + oiio::ImageBufAlgo::convolve(outBuf, inBuf, K); + + downscale(_pyramid_color[lvl + 1], _filter_buffer[lvl]); + } + + return true; +} + + +bool GaussianPyramidNoMask::downscale(image::Image & output, const image::Image & input) { + + for (int i = 0; i < output.Height(); i++) { + int ui = i * 2; + + for (int j = 0; j < output.Width(); j++) { + int uj = j * 2; + + output(i, j) = input(ui, uj); + } + } + + return true; +} + +} \ No newline at end of file diff --git a/src/aliceVision/panorama/gaussian.hpp b/src/aliceVision/panorama/gaussian.hpp new file mode 100644 index 0000000000..7f242b5547 --- /dev/null +++ b/src/aliceVision/panorama/gaussian.hpp @@ -0,0 +1,36 @@ +#pragma once + +#include + +namespace aliceVision { + +class GaussianPyramidNoMask { +public: + GaussianPyramidNoMask(const size_t width_base, const size_t height_base, const size_t limit_scales = 64); + + bool process(const image::Image & input); + + + bool downscale(image::Image & output, const image::Image & input); + + const size_t getScalesCount() const { + return _scales; + } + + const std::vector> & getPyramidColor() const { + return _pyramid_color; + } + + std::vector> & getPyramidColor() { + return _pyramid_color; + } + +protected: + std::vector> _pyramid_color; + std::vector> _filter_buffer; + size_t _width_base; + size_t _height_base; + size_t _scales; +}; + +} \ No newline at end of file diff --git a/src/aliceVision/panorama/remapBbox.cpp b/src/aliceVision/panorama/remapBbox.cpp new file mode 100644 index 0000000000..a0234047d0 --- /dev/null +++ b/src/aliceVision/panorama/remapBbox.cpp @@ -0,0 +1,393 @@ +#pragma once + + +#include "remapBbox.hpp" +#include "sphericalMapping.hpp" + +namespace aliceVision { + +bool isPoleInTriangle(const Vec3 & pt1, const Vec3 & pt2, const Vec3 & pt3) { + + double a = (pt2.x()*pt3.z() - pt3.x()*pt2.z())/(pt1.x()*pt2.z() - pt1.x()*pt3.z() - pt2.x()*pt1.z() + pt2.x()*pt3.z() + pt3.x()*pt1.z() - pt3.x()*pt2.z()); + double b = (-pt1.x()*pt3.z() + pt3.x()*pt1.z())/(pt1.x()*pt2.z() - pt1.x()*pt3.z() - pt2.x()*pt1.z() + pt2.x()*pt3.z() + pt3.x()*pt1.z() - pt3.x()*pt2.z()); + double c = 1.0 - a - b; + + if (a < 0.0 || a > 1.0) return false; + if (b < 0.0 || b > 1.0) return false; + if (c < 0.0 || c > 1.0) return false; + + return true; +} + +bool crossHorizontalLoop(const Vec3 & pt1, const Vec3 & pt2) { + Vec3 direction = pt2 - pt1; + + /*Vertical line*/ + if (std::abs(direction(0)) < 1e-12) { + return false; + } + + double t = - pt1(0) / direction(0); + Vec3 cross = pt1 + direction * t; + + if (t >= 0.0 && t <= 1.0) { + if (cross(2) < 0.0) { + return true; + } + } + + return false; +} + +Vec3 getExtremaY(const Vec3 & pt1, const Vec3 & pt2) { + + Vec3 delta = pt2 - pt1; + double dx = delta(0); + double dy = delta(1); + double dz = delta(2); + double sx = pt1(0); + double sy = pt1(1); + double sz = pt1(2); + + double ot_y = -(dx*sx*sy - (dy*sx)*(dy*sx) - (dy*sz)*(dy*sz) + dz*sy*sz)/(dx*dx*sy - dx*dy*sx - dy*dz*sz + dz*dz*sy); + + Vec3 pt_extrema = pt1 + ot_y * delta; + + return pt_extrema.normalized(); +} + +bool computeCoarseBB_Equidistant(BoundingBox & coarse_bbox, const std::pair & panoramaSize, const geometry::Pose3 & pose, const aliceVision::camera::IntrinsicBase & intrinsics) { + + const aliceVision::camera::EquiDistant & cam = dynamic_cast(intrinsics); + + + bool loop = false; + std::vector vec_bool(panoramaSize.second, false); + + for (int i = 0; i < panoramaSize.second; i++) { + + { + Vec3 ray = SphericalMapping::fromEquirectangular(Vec2(0, i), panoramaSize.first, panoramaSize.second); + + /** + * Check that this ray should be visible. + * This test is camera type dependent + */ + Vec3 transformedRay = pose(ray); + if (!intrinsics.isVisibleRay(transformedRay)) { + continue; + } + + /** + * Project this ray to camera pixel coordinates + */ + const Vec2 pix_disto = intrinsics.project(pose, ray, true); + + /** + * Ignore invalid coordinates + */ + if (!intrinsics.isVisible(pix_disto)) { + continue; + } + } + + { + Vec3 ray = SphericalMapping::fromEquirectangular(Vec2(panoramaSize.first - 1, i), panoramaSize.first, panoramaSize.second); + + /** + * Check that this ray should be visible. + * This test is camera type dependent + */ + Vec3 transformedRay = pose(ray); + if (!intrinsics.isVisibleRay(transformedRay)) { + continue; + } + + /** + * Project this ray to camera pixel coordinates + */ + const Vec2 pix_disto = intrinsics.project(pose, ray, true); + + /** + * Ignore invalid coordinates + */ + if (!intrinsics.isVisible(pix_disto)) { + continue; + } + + vec_bool[i] = true; + loop = true; + } + } + + if (vec_bool[0] || vec_bool[panoramaSize.second - 1]) { + loop = false; + } + + if (!loop) { + coarse_bbox.left = 0; + coarse_bbox.top = 0; + coarse_bbox.width = panoramaSize.first; + coarse_bbox.height = panoramaSize.second; + return true; + } + + int last_x = 0; + + for (int x = panoramaSize.first - 1; x >= 0; x--) { + + size_t count = 0; + + for (int i = 0; i < panoramaSize.second; i++) { + + if (vec_bool[i] == false) { + continue; + } + + Vec3 ray = SphericalMapping::fromEquirectangular(Vec2(x, i), panoramaSize.first, panoramaSize.second); + + /** + * Check that this ray should be visible. + * This test is camera type dependent + */ + Vec3 transformedRay = pose(ray); + if (!intrinsics.isVisibleRay(transformedRay)) { + vec_bool[i] = false; + continue; + } + + /** + * Project this ray to camera pixel coordinates + */ + const Vec2 pix_disto = intrinsics.project(pose, ray, true); + + /** + * Ignore invalid coordinates + */ + if (!intrinsics.isVisible(pix_disto)) { + vec_bool[i] = false; + continue; + } + + count++; + } + + if (count == 0) { + break; + } + + last_x = x; + } + + + coarse_bbox.left = last_x; + coarse_bbox.top = 0; + coarse_bbox.width = panoramaSize.first; + coarse_bbox.height = panoramaSize.second; + + return true; +} + +bool computeCoarseBB_Pinhole(BoundingBox & coarse_bbox, const std::pair & panoramaSize, const geometry::Pose3 & pose, const aliceVision::camera::IntrinsicBase & intrinsics) { + + int bbox_left, bbox_top; + int bbox_right, bbox_bottom; + int bbox_width, bbox_height; + + /*Estimate distorted maximal distance from optical center*/ + Vec2 pts[] = {{0.0f, 0.0f}, {intrinsics.w(), 0.0f}, {intrinsics.w(), intrinsics.h()}, {0.0f, intrinsics.h()}}; + float max_radius = 0.0; + for (int i = 0; i < 4; i++) { + + Vec2 ptmeter = intrinsics.ima2cam(pts[i]); + float radius = ptmeter.norm(); + max_radius = std::max(max_radius, radius); + } + + /* Estimate undistorted maximal distance from optical center */ + float max_radius_distorted = max_radius;//intrinsics.getMaximalDistortion(0.0, max_radius); + + /* + Coarse rectangle bouding box in camera space + We add intermediate points to ensure arclength between 2 points is never more than 180° + */ + Vec2 pts_radius[] = { + {-max_radius_distorted, -max_radius_distorted}, + {0, -max_radius_distorted}, + {max_radius_distorted, -max_radius_distorted}, + {max_radius_distorted, 0}, + {max_radius_distorted, max_radius_distorted}, + {0, max_radius_distorted}, + {-max_radius_distorted, max_radius_distorted}, + {-max_radius_distorted, 0} + }; + + + /* + Transform bounding box into the panorama frame. + Point are on a unit sphere. + */ + Vec3 rotated_pts[8]; + for (int i = 0; i < 8; i++) { + Vec3 pt3d = intrinsics.toUnitSphere(pts_radius[i]); + rotated_pts[i] = pose.rotation().transpose() * pt3d; + } + + /* Vertical Default solution : no pole*/ + bbox_top = panoramaSize.second; + bbox_bottom = 0; + + for (int i = 0; i < 8; i++) { + int i2 = (i + 1) % 8; + + Vec3 extremaY = getExtremaY(rotated_pts[i], rotated_pts[i2]); + + Vec2 res; + res = SphericalMapping::toEquirectangular(extremaY, panoramaSize.first, panoramaSize.second); + bbox_top = std::min(int(floor(res(1))), bbox_top); + bbox_bottom = std::max(int(ceil(res(1))), bbox_bottom); + + res = SphericalMapping::toEquirectangular(rotated_pts[i], panoramaSize.first, panoramaSize.second); + bbox_top = std::min(int(floor(res(1))), bbox_top); + bbox_bottom = std::max(int(ceil(res(1))), bbox_bottom); + } + + /* + Check if our region circumscribe a pole of the sphere : + Check that the region projected on the Y=0 plane contains the point (0, 0) + This is a special projection case + */ + bool pole = isPoleInTriangle(rotated_pts[0], rotated_pts[1], rotated_pts[7]); + pole |= isPoleInTriangle(rotated_pts[1], rotated_pts[2], rotated_pts[3]); + pole |= isPoleInTriangle(rotated_pts[3], rotated_pts[4], rotated_pts[5]); + pole |= isPoleInTriangle(rotated_pts[7], rotated_pts[5], rotated_pts[6]); + pole |= isPoleInTriangle(rotated_pts[1], rotated_pts[3], rotated_pts[5]); + pole |= isPoleInTriangle(rotated_pts[1], rotated_pts[5], rotated_pts[7]); + + + if (pole) { + Vec3 normal = (rotated_pts[1] - rotated_pts[0]).cross(rotated_pts[3] - rotated_pts[0]); + if (normal(1) > 0) { + //Lower pole + bbox_bottom = panoramaSize.second - 1; + } + else { + //upper pole + bbox_top = 0; + } + } + + bbox_height = bbox_bottom - bbox_top + 1; + + + /*Check if we cross the horizontal loop*/ + bool crossH = false; + for (int i = 0; i < 8; i++) { + int i2 = (i + 1) % 8; + + bool cross = crossHorizontalLoop(rotated_pts[i], rotated_pts[i2]); + crossH |= cross; + } + + if (pole) { + /*Easy : if we cross the pole, the width is full*/ + bbox_left = 0; + bbox_right = panoramaSize.first - 1; + bbox_width = bbox_right - bbox_left + 1; + } + else if (crossH) { + + int first_cross = 0; + for (int i = 0; i < 8; i++) { + int i2 = (i + 1) % 8; + bool cross = crossHorizontalLoop(rotated_pts[i], rotated_pts[i2]); + if (cross) { + first_cross = i; + break; + } + } + + bbox_left = panoramaSize.first - 1; + bbox_right = 0; + bool is_right = true; + for (int index = 0; index < 8; index++) { + + int i = (index + first_cross) % 8; + int i2 = (i + 1) % 8; + + Vec2 res_1 = SphericalMapping::toEquirectangular(rotated_pts[i], panoramaSize.first, panoramaSize.second); + Vec2 res_2 = SphericalMapping::toEquirectangular(rotated_pts[i2], panoramaSize.first, panoramaSize.second); + + /*[----right //// left-----]*/ + bool cross = crossHorizontalLoop(rotated_pts[i], rotated_pts[i2]); + if (cross) { + if (res_1(0) > res_2(0)) { /*[----res2 //// res1----]*/ + bbox_left = std::min(int(res_1(0)), bbox_left); + bbox_right = std::max(int(res_2(0)), bbox_right); + is_right = true; + } + else { /*[----res1 //// res2----]*/ + bbox_left = std::min(int(res_2(0)), bbox_left); + bbox_right = std::max(int(res_1(0)), bbox_right); + is_right = false; + } + } + else { + if (is_right) { + bbox_right = std::max(int(res_1(0)), bbox_right); + bbox_right = std::max(int(res_2(0)), bbox_right); + } + else { + bbox_left = std::min(int(res_1(0)), bbox_left); + bbox_left = std::min(int(res_2(0)), bbox_left); + } + } + } + + bbox_width = bbox_right + (panoramaSize.first - bbox_left); + } + else { + /*horizontal default solution : no border crossing, no pole*/ + bbox_left = panoramaSize.first; + bbox_right = 0; + for (int i = 0; i < 8; i++) { + Vec2 res = SphericalMapping::toEquirectangular(rotated_pts[i], panoramaSize.first, panoramaSize.second); + bbox_left = std::min(int(floor(res(0))), bbox_left); + bbox_right = std::max(int(ceil(res(0))), bbox_right); + } + bbox_width = bbox_right - bbox_left + 1; + } + + /*Assign solution to result*/ + coarse_bbox.left = bbox_left; + coarse_bbox.top = bbox_top; + coarse_bbox.width = bbox_width; + coarse_bbox.height = bbox_height; + + return true; +} + + + +bool computeCoarseBB(BoundingBox & coarse_bbox, const std::pair & panoramaSize, const geometry::Pose3 & pose, const aliceVision::camera::IntrinsicBase & intrinsics) { + + bool ret = true; + + if (isPinhole(intrinsics.getType())) { + ret = computeCoarseBB_Pinhole(coarse_bbox, panoramaSize, pose, intrinsics); + } + else if (isEquidistant(intrinsics.getType())) { + ret = computeCoarseBB_Equidistant(coarse_bbox, panoramaSize, pose, intrinsics); + } + else { + coarse_bbox.left = 0; + coarse_bbox.top = 0; + coarse_bbox.width = panoramaSize.first; + coarse_bbox.height = panoramaSize.second; + ret = true; + } + + return ret; +} + +} \ No newline at end of file diff --git a/src/aliceVision/panorama/remapBbox.hpp b/src/aliceVision/panorama/remapBbox.hpp new file mode 100644 index 0000000000..ad6594f95f --- /dev/null +++ b/src/aliceVision/panorama/remapBbox.hpp @@ -0,0 +1,13 @@ +#pragma once + +#include +#include +#include + +#include "boundingBox.hpp" + +namespace aliceVision { + +bool computeCoarseBB(BoundingBox & coarse_bbox, const std::pair & panoramaSize, const geometry::Pose3 & pose, const aliceVision::camera::IntrinsicBase & intrinsics); + +} \ No newline at end of file diff --git a/src/aliceVision/panorama/sphericalMapping.cpp b/src/aliceVision/panorama/sphericalMapping.cpp new file mode 100644 index 0000000000..f499f83b19 --- /dev/null +++ b/src/aliceVision/panorama/sphericalMapping.cpp @@ -0,0 +1,46 @@ +#include "sphericalMapping.hpp" + +namespace aliceVision +{ + +namespace SphericalMapping +{ +/** + * Map from equirectangular to spherical coordinates + * @param equirectangular equirectangular coordinates + * @param width number of pixels used to represent longitude + * @param height number of pixels used to represent latitude + * @return spherical coordinates + */ +Vec3 fromEquirectangular(const Vec2 & equirectangular, int width, int height) +{ + const double latitude = (equirectangular(1) / double(height)) * M_PI - M_PI_2; + const double longitude = ((equirectangular(0) / double(width)) * 2.0 * M_PI) - M_PI; + + const double Px = cos(latitude) * sin(longitude); + const double Py = sin(latitude); + const double Pz = cos(latitude) * cos(longitude); + + return Vec3(Px, Py, Pz); +} + +/** + * Map from Spherical to equirectangular coordinates + * @param spherical spherical coordinates + * @param width number of pixels used to represent longitude + * @param height number of pixels used to represent latitude + * @return equirectangular coordinates + */ +Vec2 toEquirectangular(const Vec3 & spherical, int width, int height) { + + double vertical_angle = asin(spherical(1)); + double horizontal_angle = atan2(spherical(0), spherical(2)); + + double latitude = ((vertical_angle + M_PI_2) / M_PI) * height; + double longitude = ((horizontal_angle + M_PI) / (2.0 * M_PI)) * width; + + return Vec2(longitude, latitude); +} + +} +} \ No newline at end of file diff --git a/src/aliceVision/panorama/sphericalMapping.hpp b/src/aliceVision/panorama/sphericalMapping.hpp new file mode 100644 index 0000000000..968dd2baab --- /dev/null +++ b/src/aliceVision/panorama/sphericalMapping.hpp @@ -0,0 +1,29 @@ +#pragma once + +#include + +namespace aliceVision +{ + +namespace SphericalMapping +{ +/** + * Map from equirectangular to spherical coordinates + * @param equirectangular equirectangular coordinates + * @param width number of pixels used to represent longitude + * @param height number of pixels used to represent latitude + * @return spherical coordinates + */ +Vec3 fromEquirectangular(const Vec2 & equirectangular, int width, int height); + +/** + * Map from Spherical to equirectangular coordinates + * @param spherical spherical coordinates + * @param width number of pixels used to represent longitude + * @param height number of pixels used to represent latitude + * @return equirectangular coordinates + */ +Vec2 toEquirectangular(const Vec3 & spherical, int width, int height); + +} +} \ No newline at end of file diff --git a/src/aliceVision/panorama/warper.cpp b/src/aliceVision/panorama/warper.cpp new file mode 100644 index 0000000000..760b3bb9cb --- /dev/null +++ b/src/aliceVision/panorama/warper.cpp @@ -0,0 +1,116 @@ +#include "warper.hpp" + +namespace aliceVision { + +bool Warper::warp(const CoordinatesMap & map, const aliceVision::image::Image & source) { + + /** + * Copy additional info from map + */ + _offset_x = map.getOffsetX(); + _offset_y = map.getOffsetY(); + _mask = map.getMask(); + + const image::Sampler2d sampler; + const aliceVision::image::Image & coordinates = map.getCoordinates(); + + /** + * Create buffer + * No longer need to keep a 2**x size + */ + _color = aliceVision::image::Image(coordinates.Width(), coordinates.Height()); + + /** + * Simple warp + */ + for (size_t i = 0; i < _color.Height(); i++) { + for (size_t j = 0; j < _color.Width(); j++) { + + bool valid = _mask(i, j); + if (!valid) { + continue; + } + + const Eigen::Vector2d & coord = coordinates(i, j); + const image::RGBfColor pixel = sampler(source, coord(1), coord(0)); + + _color(i, j) = pixel; + } + } + + return true; +} + +bool GaussianWarper::warp(const CoordinatesMap & map, const GaussianPyramidNoMask & pyramid) { + + /** + * Copy additional info from map + */ + _offset_x = map.getOffsetX(); + _offset_y = map.getOffsetY(); + _mask = map.getMask(); + + const image::Sampler2d sampler; + const aliceVision::image::Image & coordinates = map.getCoordinates(); + + /** + * Create a pyramid for input + */ + const std::vector> & mlsource = pyramid.getPyramidColor(); + size_t max_level = pyramid.getScalesCount() - 1; + + /** + * Create buffer + */ + _color = aliceVision::image::Image(coordinates.Width(), coordinates.Height(), true, image::RGBfColor(1.0, 0.0, 0.0)); + + /** + * Multi level warp + */ + for (size_t i = 0; i < _color.Height(); i++) { + for (size_t j = 0; j < _color.Width(); j++) { + + bool valid = _mask(i, j); + if (!valid) { + continue; + } + + if (i == _color.Height() - 1 || j == _color.Width() - 1 || !_mask(i + 1, j) || !_mask(i, j + 1)) { + const Eigen::Vector2d & coord = coordinates(i, j); + const image::RGBfColor pixel = sampler(mlsource[0], coord(1), coord(0)); + _color(i, j) = pixel; + continue; + } + + const Eigen::Vector2d & coord_mm = coordinates(i, j); + const Eigen::Vector2d & coord_mp = coordinates(i, j + 1); + const Eigen::Vector2d & coord_pm = coordinates(i + 1, j); + + double dxx = coord_pm(0) - coord_mm(0); + double dxy = coord_mp(0) - coord_mm(0); + double dyx = coord_pm(1) - coord_mm(1); + double dyy = coord_mp(1) - coord_mm(1); + double det = std::abs(dxx*dyy - dxy*dyx); + double scale = sqrt(det); + + double flevel = std::max(0.0, log2(scale)); + size_t blevel = std::min(max_level, size_t(floor(flevel))); + + double dscale, x, y; + dscale = 1.0 / pow(2.0, blevel); + x = coord_mm(0) * dscale; + y = coord_mm(1) * dscale; + /*Fallback to first level if outside*/ + if (x >= mlsource[blevel].Width() - 1 || y >= mlsource[blevel].Height() - 1) { + _color(i, j) = sampler(mlsource[0], coord_mm(1), coord_mm(0)); + continue; + } + + _color(i, j) = sampler(mlsource[blevel], y, x); + } + } + + return true; + } + + } \ No newline at end of file diff --git a/src/aliceVision/panorama/warper.hpp b/src/aliceVision/panorama/warper.hpp new file mode 100644 index 0000000000..d7f15bca15 --- /dev/null +++ b/src/aliceVision/panorama/warper.hpp @@ -0,0 +1,43 @@ +#pragma once + +#include "coordinatesMap.hpp" +#include "gaussian.hpp" + + +namespace aliceVision { + +class Warper { +public: + virtual bool warp(const CoordinatesMap & map, const aliceVision::image::Image & source); + + const aliceVision::image::Image & getColor() const { + return _color; + } + + const aliceVision::image::Image & getMask() const { + return _mask; + } + + + size_t getOffsetX() const { + return _offset_x; + } + + size_t getOffsetY() const { + return _offset_y; + } + +protected: + size_t _offset_x = 0; + size_t _offset_y = 0; + + aliceVision::image::Image _color; + aliceVision::image::Image _mask; +}; + +class GaussianWarper : public Warper { +public: + virtual bool warp(const CoordinatesMap & map, const GaussianPyramidNoMask & pyramid); +}; + +} \ No newline at end of file diff --git a/src/software/pipeline/CMakeLists.txt b/src/software/pipeline/CMakeLists.txt index 7d46dddba3..aae1c1b152 100644 --- a/src/software/pipeline/CMakeLists.txt +++ b/src/software/pipeline/CMakeLists.txt @@ -134,6 +134,7 @@ if(ALICEVISION_BUILD_SFM) aliceVision_sfm aliceVision_sfmData aliceVision_sfmDataIO + aliceVision_panorama ${Boost_LIBRARIES} ) alicevision_add_software(aliceVision_panoramaCompositing From 3fb2acb0ffc0cc62754c233b9628f254e8afc183 Mon Sep 17 00:00:00 2001 From: Fabien Servant Date: Wed, 30 Sep 2020 15:07:37 +0200 Subject: [PATCH 02/79] Tiled warping --- src/aliceVision/panorama/boundingBox.hpp | 53 +- src/aliceVision/panorama/coordinatesMap.cpp | 58 +- src/aliceVision/panorama/coordinatesMap.hpp | 13 + src/aliceVision/panorama/distance.cpp | 9 +- src/aliceVision/panorama/distance.hpp | 2 +- .../pipeline/main_panoramaWarping.cpp | 1598 ++++------------- 6 files changed, 516 insertions(+), 1217 deletions(-) diff --git a/src/aliceVision/panorama/boundingBox.hpp b/src/aliceVision/panorama/boundingBox.hpp index 920efa0952..780e3b7dde 100644 --- a/src/aliceVision/panorama/boundingBox.hpp +++ b/src/aliceVision/panorama/boundingBox.hpp @@ -2,20 +2,67 @@ struct BoundingBox { + int left; int top; int width; int height; - BoundingBox() = default; + BoundingBox() { + left = -1; + top = -1; + width = 0; + height = 0; + }; BoundingBox(int l, int t, int w, int h) : left(l), top(t), width(w), height(h) {} int getRight() const { - return left + width; + return left + width - 1; } int getBottom() const { - return top + height; + return top + height - 1; + } + + void snapToGrid(uint32_t gridSize) { + + int right = getRight(); + int bottom = getBottom(); + + int leftBounded = int(floor(double(left) / double(gridSize))) * int(gridSize); + int topBounded = int(floor(double(top) / double(gridSize))) * int(gridSize); + int widthBounded = int(ceil(double(right - leftBounded + 1) / double(gridSize))) * int(gridSize); + int heightBounded = int(ceil(double(bottom - topBounded + 1) / double(gridSize))) * int(gridSize); + + left = leftBounded; + top = topBounded; + width = widthBounded; + height = heightBounded; + } + + void unionWith(const BoundingBox & other) { + + if (left < 0 && top < 0) { + left = other.left; + top = other.top; + width = other.width; + height = other.height; + return; + } + + int rt = getRight(); + int ro = other.getRight(); + int bt = getBottom(); + int bo = other.getBottom(); + + left = std::min(left, other.left); + top = std::min(top, other.top); + + int maxr = std::max(rt, ro); + int maxb = std::max(bt, bo); + + width = maxr - left + 1; + height = maxb - top + 1; } }; \ No newline at end of file diff --git a/src/aliceVision/panorama/coordinatesMap.cpp b/src/aliceVision/panorama/coordinatesMap.cpp index aac3380438..9cd3b00d98 100644 --- a/src/aliceVision/panorama/coordinatesMap.cpp +++ b/src/aliceVision/panorama/coordinatesMap.cpp @@ -13,13 +13,15 @@ bool CoordinatesMap::build(const std::pair & panoramaSize, const geome _coordinates = aliceVision::image::Image(coarseBbox.width, coarseBbox.height, false); _mask = aliceVision::image::Image(coarseBbox.width, coarseBbox.height, true, 0); + size_t max_x = 0; + size_t max_y = 0; + size_t min_x = panoramaSize.first; + size_t min_y = panoramaSize.second; - #pragma omp parallel for for (size_t y = 0; y < coarseBbox.height; y++) { size_t cy = y + coarseBbox.top; - for (size_t x = 0; x < coarseBbox.width; x++) { size_t cx = x + coarseBbox.left; @@ -49,15 +51,67 @@ bool CoordinatesMap::build(const std::pair & panoramaSize, const geome _coordinates(y, x) = pix_disto; _mask(y, x) = 1; + + min_x = std::min(cx, min_x); + min_y = std::min(cy, min_y); + max_x = std::max(cx, max_x); + max_y = std::max(cy, max_y); } } _offset_x = coarseBbox.left; _offset_y = coarseBbox.top; + _boundingBox.left = min_x; + _boundingBox.top = min_y; + _boundingBox.width = max_x - min_x + 1; + _boundingBox.height = max_y - min_y + 1; + return true; } + +bool CoordinatesMap::containsPixels(const std::pair & panoramaSize, const geometry::Pose3 & pose, const aliceVision::camera::IntrinsicBase & intrinsics, const BoundingBox &coarseBbox) { + + for (size_t y = 0; y < coarseBbox.height; y++) { + + size_t cy = y + coarseBbox.top; + + + for (size_t x = 0; x < coarseBbox.width; x++) { + + size_t cx = x + coarseBbox.left; + + Vec3 ray = SphericalMapping::fromEquirectangular(Vec2(cx, cy), panoramaSize.first, panoramaSize.second); + + /** + * Check that this ray should be visible. + * This test is camera type dependent + */ + Vec3 transformedRay = pose(ray); + if (!intrinsics.isVisibleRay(transformedRay)) { + continue; + } + + /** + * Project this ray to camera pixel coordinates + */ + const Vec2 pix_disto = intrinsics.project(pose, ray, true); + + /** + * Ignore invalid coordinates + */ + if (!intrinsics.isVisible(pix_disto)) { + continue; + } + + return true; + } + } + + return false; +} + bool CoordinatesMap::computeScale(double & result, float ratioUpscale) { std::vector scales; diff --git a/src/aliceVision/panorama/coordinatesMap.hpp b/src/aliceVision/panorama/coordinatesMap.hpp index d07cc3020d..f088d80167 100644 --- a/src/aliceVision/panorama/coordinatesMap.hpp +++ b/src/aliceVision/panorama/coordinatesMap.hpp @@ -18,6 +18,14 @@ class CoordinatesMap { */ bool build(const std::pair & panoramaSize, const geometry::Pose3 & pose, const aliceVision::camera::IntrinsicBase & intrinsics, const BoundingBox &coarseBbox); + /** + * Check if there is any valid pixel + * @param panoramaSize desired output panoramaSize + * @param pose the camera pose wrt an arbitrary reference frame + * @param intrinsics the camera intrinsics + */ + bool containsPixels(const std::pair & panoramaSize, const geometry::Pose3 & pose, const aliceVision::camera::IntrinsicBase & intrinsics, const BoundingBox &coarseBbox); + bool computeScale(double & result, float ratioUpscale); size_t getOffsetX() const { @@ -28,6 +36,10 @@ class CoordinatesMap { return _offset_y; } + BoundingBox getBoundingBox() const { + return _boundingBox; + } + const aliceVision::image::Image & getCoordinates() const { return _coordinates; } @@ -42,6 +54,7 @@ class CoordinatesMap { aliceVision::image::Image _coordinates; aliceVision::image::Image _mask; + BoundingBox _boundingBox; }; } \ No newline at end of file diff --git a/src/aliceVision/panorama/distance.cpp b/src/aliceVision/panorama/distance.cpp index 88b54b90e5..2a7d7504c8 100644 --- a/src/aliceVision/panorama/distance.cpp +++ b/src/aliceVision/panorama/distance.cpp @@ -4,16 +4,13 @@ namespace aliceVision { -bool distanceToCenter(aliceVision::image::Image & _weights, const CoordinatesMap & map) { +bool distanceToCenter(aliceVision::image::Image & _weights, const CoordinatesMap & map, int width, int height) { const aliceVision::image::Image & coordinates = map.getCoordinates(); const aliceVision::image::Image & mask = map.getMask(); - float w = static_cast(coordinates.Width()); - float h = static_cast(coordinates.Height()); - - float cx = w / 2.0f; - float cy = h / 2.0f; + float cx = width / 2.0f; + float cy = height / 2.0f; _weights = aliceVision::image::Image(coordinates.Width(), coordinates.Height()); diff --git a/src/aliceVision/panorama/distance.hpp b/src/aliceVision/panorama/distance.hpp index 781607ca9d..b27dd9dcd9 100644 --- a/src/aliceVision/panorama/distance.hpp +++ b/src/aliceVision/panorama/distance.hpp @@ -6,6 +6,6 @@ namespace aliceVision { -bool distanceToCenter(aliceVision::image::Image & _weights, const CoordinatesMap & map); +bool distanceToCenter(aliceVision::image::Image & _weights, const CoordinatesMap & map, int width, int height); } \ No newline at end of file diff --git a/src/software/pipeline/main_panoramaWarping.cpp b/src/software/pipeline/main_panoramaWarping.cpp index 57901bcbf0..f9d67d4e7b 100644 --- a/src/software/pipeline/main_panoramaWarping.cpp +++ b/src/software/pipeline/main_panoramaWarping.cpp @@ -4,28 +4,24 @@ // v. 2.0. If a copy of the MPL was not distributed with this file, // You can obtain one at https://mozilla.org/MPL/2.0/. -// Input and geometry -#include -#include - -// Image stuff -#include -#include - -// Logging stuff -#include - // Reading command line options #include +#include #include #include -// IO -#include -#include -#include -#include -#include +// Image related +#include + +// Sfmdata +#include +#include + +// Internal functions +#include +#include +#include +#include // These constants define the current software version. // They must be updated when the command line is changed. @@ -35,1198 +31,390 @@ using namespace aliceVision; namespace po = boost::program_options; -namespace bpt = boost::property_tree; namespace fs = boost::filesystem; -namespace SphericalMapping +bool computeOptimalPanoramaSize(std::pair& optimalSize, const sfmData::SfMData& sfmData, const float ratioUpscale) { - /** - * Map from equirectangular to spherical coordinates - * @param equirectangular equirectangular coordinates - * @param width number of pixels used to represent longitude - * @param height number of pixels used to represent latitude - * @return spherical coordinates - */ - Vec3 fromEquirectangular(const Vec2 & equirectangular, int width, int height) - { - const double latitude = (equirectangular(1) / double(height)) * M_PI - M_PI_2; - const double longitude = ((equirectangular(0) / double(width)) * 2.0 * M_PI) - M_PI; - - const double Px = cos(latitude) * sin(longitude); - const double Py = sin(latitude); - const double Pz = cos(latitude) * cos(longitude); - - return Vec3(Px, Py, Pz); - } - - /** - * Map from Spherical to equirectangular coordinates - * @param spherical spherical coordinates - * @param width number of pixels used to represent longitude - * @param height number of pixels used to represent latitude - * @return equirectangular coordinates - */ - Vec2 toEquirectangular(const Vec3 & spherical, int width, int height) { - - double vertical_angle = asin(spherical(1)); - double horizontal_angle = atan2(spherical(0), spherical(2)); - - double latitude = ((vertical_angle + M_PI_2) / M_PI) * height; - double longitude = ((horizontal_angle + M_PI) / (2.0 * M_PI)) * width; - - return Vec2(longitude, latitude); - } - - /** - * Map from Spherical to equirectangular coordinates in radians - * @param spherical spherical coordinates - * @return equirectangular coordinates - */ - Vec2 toLongitudeLatitude(const Vec3 & spherical) { - - double latitude = asin(spherical(1)); - double longitude = atan2(spherical(0), spherical(2)); - - return Vec2(longitude, latitude); - } -} - -class GaussianPyramidNoMask { -public: - GaussianPyramidNoMask(const size_t width_base, const size_t height_base, const size_t limit_scales = 64) : - _width_base(width_base), - _height_base(height_base) - { - /** - * Compute optimal scale - * The smallest level will be at least of size min_size - */ - size_t min_dim = std::min(_width_base, _height_base); - size_t min_size = 32; - _scales = std::min(limit_scales, static_cast(floor(log2(double(min_dim) / float(min_size))))); - - - /** - * Create pyramid - **/ - size_t new_width = _width_base; - size_t new_height = _height_base; - for (int i = 0; i < _scales; i++) { - - _pyramid_color.push_back(image::Image(new_width, new_height, true, image::RGBfColor(0))); - _filter_buffer.push_back(image::Image(new_width, new_height, true, image::RGBfColor(0))); - new_height /= 2; - new_width /= 2; - } - } - - bool process(const image::Image & input) { - - if (input.Height() != _pyramid_color[0].Height()) return false; - if (input.Width() != _pyramid_color[0].Width()) return false; - - - /** - * Kernel - */ - oiio::ImageBuf K; - oiio::ImageBufAlgo::make_kernel(K, "gaussian", 5, 5); - - /** - * Build pyramid - */ - _pyramid_color[0] = input; - for (int lvl = 0; lvl < _scales - 1; lvl++) { - - const image::Image & source = _pyramid_color[lvl]; - image::Image & dst = _filter_buffer[lvl]; - - oiio::ImageSpec spec(source.Width(), source.Height(), 3, oiio::TypeDesc::FLOAT); - - const oiio::ImageBuf inBuf(spec, const_cast(source.data())); - oiio::ImageBuf outBuf(spec, dst.data()); - oiio::ImageBufAlgo::convolve(outBuf, inBuf, K); - - downscale(_pyramid_color[lvl + 1], _filter_buffer[lvl]); - } - - return true; - } - - - bool downscale(image::Image & output, const image::Image & input) { - - for (int i = 0; i < output.Height(); i++) { - int ui = i * 2; - - for (int j = 0; j < output.Width(); j++) { - int uj = j * 2; - - output(i, j) = input(ui, uj); - } - } - - return true; - } - - const size_t getScalesCount() const { - return _scales; - } - - const std::vector> & getPyramidColor() const { - return _pyramid_color; - } - - std::vector> & getPyramidColor() { - return _pyramid_color; - } - -protected: - std::vector> _pyramid_color; - std::vector> _filter_buffer; - size_t _width_base; - size_t _height_base; - size_t _scales; -}; - -class CoordinatesMap { -private: - struct BBox { - int left; - int top; - int width; - int height; - }; - -public: - /** - * Build coordinates map given camera properties - * @param panoramaSize desired output panoramaSize - * @param pose the camera pose wrt an arbitrary reference frame - * @param intrinsics the camera intrinsics - */ - bool build(const std::pair & panoramaSize, const geometry::Pose3 & pose, const aliceVision::camera::IntrinsicBase & intrinsics) { - - BBox coarse_bbox; - if (!computeCoarseBB(coarse_bbox, panoramaSize, pose, intrinsics)) { - return false; - } - - - /* Effectively compute the warping map */ - aliceVision::image::Image buffer_coordinates(coarse_bbox.width, coarse_bbox.height, false); - aliceVision::image::Image buffer_mask(coarse_bbox.width, coarse_bbox.height, true, 0); - - size_t max_x = 0; - size_t max_y = 0; - size_t min_x = panoramaSize.first; - size_t min_y = panoramaSize.second; - -#ifdef _MSC_VER - // TODO - // no support for reduction min in MSVC implementation of openmp -#else - #pragma omp parallel for reduction(min: min_x, min_y) reduction(max: max_x, max_y) -#endif - for (size_t y = 0; y < coarse_bbox.height; y++) { - - size_t cy = y + coarse_bbox.top; - - size_t row_max_x = 0; - size_t row_max_y = 0; - size_t row_min_x = panoramaSize.first; - size_t row_min_y = panoramaSize.second; - - - for (size_t x = 0; x < coarse_bbox.width; x++) { - - size_t cx = x + coarse_bbox.left; - - Vec3 ray = SphericalMapping::fromEquirectangular(Vec2(cx, cy), panoramaSize.first, panoramaSize.second); - - /** - * Check that this ray should be visible. - * This test is camera type dependent - */ - Vec3 transformedRay = pose(ray); - if (!intrinsics.isVisibleRay(transformedRay)) { - continue; - } - - /** - * Project this ray to camera pixel coordinates - */ - const Vec2 pix_disto = intrinsics.project(pose, ray, true); - - /** - * Ignore invalid coordinates - */ - if (!intrinsics.isVisible(pix_disto)) { - continue; - } - - buffer_coordinates(y, x) = pix_disto; - buffer_mask(y, x) = 1; - - row_min_x = std::min(x, row_min_x); - row_min_y = std::min(y, row_min_y); - row_max_x = std::max(x, row_max_x); - row_max_y = std::max(y, row_max_y); - } - - min_x = std::min(row_min_x, min_x); - min_y = std::min(row_min_y, min_y); - max_x = std::max(row_max_x, max_x); - max_y = std::max(row_max_y, max_y); - } - - _offset_x = coarse_bbox.left + min_x; - if (_offset_x > panoramaSize.first) { - /*The coarse bounding box may cross the borders where as the true coordinates may not*/ - int ox = int(_offset_x) - int(panoramaSize.first); - _offset_x = ox; - } - _offset_y = coarse_bbox.top + min_y; - - size_t real_width = max_x - min_x + 1; - size_t real_height = max_y - min_y + 1; - - /* Resize buffers */ - _coordinates = aliceVision::image::Image(real_width, real_height, false); - _mask = aliceVision::image::Image(real_width, real_height, true, 0); - - _coordinates.block(0, 0, real_height, real_width) = buffer_coordinates.block(min_y, min_x, real_height, real_width); - _mask.block(0, 0, real_height, real_width) = buffer_mask.block(min_y, min_x, real_height, real_width); - - return true; - } - - bool computeScale(double & result, float ratioUpscale) { - - std::vector scales; - size_t real_height = _coordinates.Height(); - size_t real_width = _coordinates.Width(); - - for (int i = 1; i < real_height - 2; i++) { - for (int j = 1; j < real_width - 2; j++) { - if (!_mask(i, j) || !_mask(i, j + 1) || !_mask(i + 1, j)) { - continue; - } - - double dxx = _coordinates(i, j + 1).x() - _coordinates(i, j).x(); - double dxy = _coordinates(i + 1, j).x() - _coordinates(i, j).x(); - double dyx = _coordinates(i, j + 1).y() - _coordinates(i, j).y(); - double dyy = _coordinates(i + 1, j).y() - _coordinates(i, j).y(); - - double det = std::abs(dxx*dyy - dxy*dyx); - if (det < 1e-12) continue; - - scales.push_back(det); - } - } - - if (scales.empty()) return false; - - std::sort(scales.begin(), scales.end()); - const float indexRatio = float(scales.size() - 1) * ratioUpscale; - float intPart = 0.0f; - const float fractPart = std::modf(indexRatio, &intPart); - const size_t indexL = int(intPart); - const size_t indexU = std::min(scales.size() - 1, size_t(std::ceil(indexRatio))); - const double scale = scales[indexL] * (1.0f - fractPart) + scales[indexU] * fractPart; - result = std::sqrt(scale); - - return true; - } - - size_t getOffsetX() const { - return _offset_x; - } - - size_t getOffsetY() const { - return _offset_y; - } - - const aliceVision::image::Image & getCoordinates() const { - return _coordinates; - } - - const aliceVision::image::Image & getMask() const { - return _mask; - } - -private: - - bool computeCoarseBB(BBox & coarse_bbox, const std::pair & panoramaSize, const geometry::Pose3 & pose, const aliceVision::camera::IntrinsicBase & intrinsics) { - - bool ret = true; - - if (isPinhole(intrinsics.getType())) { - ret = computeCoarseBB_Pinhole(coarse_bbox, panoramaSize, pose, intrinsics); - } - else if (isEquidistant(intrinsics.getType())) { - ret = computeCoarseBB_Equidistant(coarse_bbox, panoramaSize, pose, intrinsics); - } - else { - coarse_bbox.left = 0; - coarse_bbox.top = 0; - coarse_bbox.width = panoramaSize.first; - coarse_bbox.height = panoramaSize.second; - ret = true; - } - - return ret; - } - - bool computeCoarseBB_Equidistant(BBox & coarse_bbox, const std::pair & panoramaSize, const geometry::Pose3 & pose, const aliceVision::camera::IntrinsicBase & intrinsics) { - - const aliceVision::camera::EquiDistant & cam = dynamic_cast(intrinsics); - - - bool loop = false; - std::vector vec_bool(panoramaSize.second, false); - - for (int i = 0; i < panoramaSize.second; i++) { - - { - Vec3 ray = SphericalMapping::fromEquirectangular(Vec2(0, i), panoramaSize.first, panoramaSize.second); - - /** - * Check that this ray should be visible. - * This test is camera type dependent - */ - Vec3 transformedRay = pose(ray); - if (!intrinsics.isVisibleRay(transformedRay)) { - continue; - } - - /** - * Project this ray to camera pixel coordinates - */ - const Vec2 pix_disto = intrinsics.project(pose, ray, true); - - /** - * Ignore invalid coordinates - */ - if (!intrinsics.isVisible(pix_disto)) { - continue; - } - } - - { - Vec3 ray = SphericalMapping::fromEquirectangular(Vec2(panoramaSize.first - 1, i), panoramaSize.first, panoramaSize.second); - - /** - * Check that this ray should be visible. - * This test is camera type dependent - */ - Vec3 transformedRay = pose(ray); - if (!intrinsics.isVisibleRay(transformedRay)) { - continue; - } - - /** - * Project this ray to camera pixel coordinates - */ - const Vec2 pix_disto = intrinsics.project(pose, ray, true); - - /** - * Ignore invalid coordinates - */ - if (!intrinsics.isVisible(pix_disto)) { - continue; - } - - vec_bool[i] = true; - loop = true; - } - } - - if (vec_bool[0] || vec_bool[panoramaSize.second - 1]) { - loop = false; - } - - if (!loop) { - coarse_bbox.left = 0; - coarse_bbox.top = 0; - coarse_bbox.width = panoramaSize.first; - coarse_bbox.height = panoramaSize.second; - return true; - } - - int last_x = 0; - - for (int x = panoramaSize.first - 1; x >= 0; x--) { - - size_t count = 0; - - for (int i = 0; i < panoramaSize.second; i++) { - - if (vec_bool[i] == false) { - continue; - } - - Vec3 ray = SphericalMapping::fromEquirectangular(Vec2(x, i), panoramaSize.first, panoramaSize.second); - - /** - * Check that this ray should be visible. - * This test is camera type dependent - */ - Vec3 transformedRay = pose(ray); - if (!intrinsics.isVisibleRay(transformedRay)) { - vec_bool[i] = false; - continue; - } - - /** - * Project this ray to camera pixel coordinates - */ - const Vec2 pix_disto = intrinsics.project(pose, ray, true); - - /** - * Ignore invalid coordinates - */ - if (!intrinsics.isVisible(pix_disto)) { - vec_bool[i] = false; - continue; - } - - count++; - } - - if (count == 0) { - break; - } - - last_x = x; - } - - - coarse_bbox.left = last_x; - coarse_bbox.top = 0; - coarse_bbox.width = panoramaSize.first; - coarse_bbox.height = panoramaSize.second; - - return true; - } - - bool computeCoarseBB_Pinhole(BBox & coarse_bbox, const std::pair & panoramaSize, const geometry::Pose3 & pose, const aliceVision::camera::IntrinsicBase & intrinsics) { - - int bbox_left, bbox_top; - int bbox_right, bbox_bottom; - int bbox_width, bbox_height; - - /*Estimate distorted maximal distance from optical center*/ - Vec2 pts[] = {{0.0f, 0.0f}, {intrinsics.w(), 0.0f}, {intrinsics.w(), intrinsics.h()}, {0.0f, intrinsics.h()}}; - float max_radius = 0.0; - for (int i = 0; i < 4; i++) { - - Vec2 ptmeter = intrinsics.ima2cam(pts[i]); - float radius = ptmeter.norm(); - max_radius = std::max(max_radius, radius); - } - - /* Estimate undistorted maximal distance from optical center */ - float max_radius_distorted = max_radius;//intrinsics.getMaximalDistortion(0.0, max_radius); - - /* - Coarse rectangle bouding box in camera space - We add intermediate points to ensure arclength between 2 points is never more than 180° - */ - Vec2 pts_radius[] = { - {-max_radius_distorted, -max_radius_distorted}, - {0, -max_radius_distorted}, - {max_radius_distorted, -max_radius_distorted}, - {max_radius_distorted, 0}, - {max_radius_distorted, max_radius_distorted}, - {0, max_radius_distorted}, - {-max_radius_distorted, max_radius_distorted}, - {-max_radius_distorted, 0} - }; - - - /* - Transform bounding box into the panorama frame. - Point are on a unit sphere. - */ - Vec3 rotated_pts[8]; - for (int i = 0; i < 8; i++) { - Vec3 pt3d = intrinsics.toUnitSphere(pts_radius[i]); - rotated_pts[i] = pose.rotation().transpose() * pt3d; - } - - /* Vertical Default solution : no pole*/ - bbox_top = panoramaSize.second; - bbox_bottom = 0; - - for (int i = 0; i < 8; i++) { - int i2 = (i + 1) % 8; - - Vec3 extremaY = getExtremaY(rotated_pts[i], rotated_pts[i2]); - - Vec2 res; - res = SphericalMapping::toEquirectangular(extremaY, panoramaSize.first, panoramaSize.second); - bbox_top = std::min(int(floor(res(1))), bbox_top); - bbox_bottom = std::max(int(ceil(res(1))), bbox_bottom); - - res = SphericalMapping::toEquirectangular(rotated_pts[i], panoramaSize.first, panoramaSize.second); - bbox_top = std::min(int(floor(res(1))), bbox_top); - bbox_bottom = std::max(int(ceil(res(1))), bbox_bottom); - } - - /* - Check if our region circumscribe a pole of the sphere : - Check that the region projected on the Y=0 plane contains the point (0, 0) - This is a special projection case - */ - bool pole = isPoleInTriangle(rotated_pts[0], rotated_pts[1], rotated_pts[7]); - pole |= isPoleInTriangle(rotated_pts[1], rotated_pts[2], rotated_pts[3]); - pole |= isPoleInTriangle(rotated_pts[3], rotated_pts[4], rotated_pts[5]); - pole |= isPoleInTriangle(rotated_pts[7], rotated_pts[5], rotated_pts[6]); - pole |= isPoleInTriangle(rotated_pts[1], rotated_pts[3], rotated_pts[5]); - pole |= isPoleInTriangle(rotated_pts[1], rotated_pts[5], rotated_pts[7]); - - - if (pole) { - Vec3 normal = (rotated_pts[1] - rotated_pts[0]).cross(rotated_pts[3] - rotated_pts[0]); - if (normal(1) > 0) { - //Lower pole - bbox_bottom = panoramaSize.second - 1; - } - else { - //upper pole - bbox_top = 0; - } - } - - bbox_height = bbox_bottom - bbox_top + 1; - - - /*Check if we cross the horizontal loop*/ - bool crossH = false; - for (int i = 0; i < 8; i++) { - int i2 = (i + 1) % 8; - - bool cross = crossHorizontalLoop(rotated_pts[i], rotated_pts[i2]); - crossH |= cross; - } - - if (pole) { - /*Easy : if we cross the pole, the width is full*/ - bbox_left = 0; - bbox_right = panoramaSize.first - 1; - bbox_width = bbox_right - bbox_left + 1; - } - else if (crossH) { - - int first_cross = 0; - for (int i = 0; i < 8; i++) { - int i2 = (i + 1) % 8; - bool cross = crossHorizontalLoop(rotated_pts[i], rotated_pts[i2]); - if (cross) { - first_cross = i; - break; - } - } - - bbox_left = panoramaSize.first - 1; - bbox_right = 0; - bool is_right = true; - for (int index = 0; index < 8; index++) { - - int i = (index + first_cross) % 8; - int i2 = (i + 1) % 8; - - Vec2 res_1 = SphericalMapping::toEquirectangular(rotated_pts[i], panoramaSize.first, panoramaSize.second); - Vec2 res_2 = SphericalMapping::toEquirectangular(rotated_pts[i2], panoramaSize.first, panoramaSize.second); - - /*[----right //// left-----]*/ - bool cross = crossHorizontalLoop(rotated_pts[i], rotated_pts[i2]); - if (cross) { - if (res_1(0) > res_2(0)) { /*[----res2 //// res1----]*/ - bbox_left = std::min(int(res_1(0)), bbox_left); - bbox_right = std::max(int(res_2(0)), bbox_right); - is_right = true; - } - else { /*[----res1 //// res2----]*/ - bbox_left = std::min(int(res_2(0)), bbox_left); - bbox_right = std::max(int(res_1(0)), bbox_right); - is_right = false; - } - } - else { - if (is_right) { - bbox_right = std::max(int(res_1(0)), bbox_right); - bbox_right = std::max(int(res_2(0)), bbox_right); - } - else { - bbox_left = std::min(int(res_1(0)), bbox_left); - bbox_left = std::min(int(res_2(0)), bbox_left); - } - } - } - - bbox_width = bbox_right + (panoramaSize.first - bbox_left); - } - else { - /*horizontal default solution : no border crossing, no pole*/ - bbox_left = panoramaSize.first; - bbox_right = 0; - for (int i = 0; i < 8; i++) { - Vec2 res = SphericalMapping::toEquirectangular(rotated_pts[i], panoramaSize.first, panoramaSize.second); - bbox_left = std::min(int(floor(res(0))), bbox_left); - bbox_right = std::max(int(ceil(res(0))), bbox_right); - } - bbox_width = bbox_right - bbox_left + 1; - } - - /*Assign solution to result*/ - coarse_bbox.left = bbox_left; - coarse_bbox.top = bbox_top; - coarse_bbox.width = bbox_width; - coarse_bbox.height = bbox_height; - - - return true; - } - - Vec3 getExtremaY(const Vec3 & pt1, const Vec3 & pt2) { - Vec3 delta = pt2 - pt1; - double dx = delta(0); - double dy = delta(1); - double dz = delta(2); - double sx = pt1(0); - double sy = pt1(1); - double sz = pt1(2); - - double ot_y = -(dx*sx*sy - (dy*sx)*(dy*sx) - (dy*sz)*(dy*sz) + dz*sy*sz)/(dx*dx*sy - dx*dy*sx - dy*dz*sz + dz*dz*sy); - - Vec3 pt_extrema = pt1 + ot_y * delta; - - return pt_extrema.normalized(); - } - - bool crossHorizontalLoop(const Vec3 & pt1, const Vec3 & pt2) { - Vec3 direction = pt2 - pt1; - - /*Vertical line*/ - if (std::abs(direction(0)) < 1e-12) { - return false; - } - - double t = - pt1(0) / direction(0); - Vec3 cross = pt1 + direction * t; - - if (t >= 0.0 && t <= 1.0) { - if (cross(2) < 0.0) { - return true; - } - } - - return false; - } - - bool isPoleInTriangle(const Vec3 & pt1, const Vec3 & pt2, const Vec3 & pt3) { - - double a = (pt2.x()*pt3.z() - pt3.x()*pt2.z())/(pt1.x()*pt2.z() - pt1.x()*pt3.z() - pt2.x()*pt1.z() + pt2.x()*pt3.z() + pt3.x()*pt1.z() - pt3.x()*pt2.z()); - double b = (-pt1.x()*pt3.z() + pt3.x()*pt1.z())/(pt1.x()*pt2.z() - pt1.x()*pt3.z() - pt2.x()*pt1.z() + pt2.x()*pt3.z() + pt3.x()*pt1.z() - pt3.x()*pt2.z()); - double c = 1.0 - a - b; - - if (a < 0.0 || a > 1.0) return false; - if (b < 0.0 || b > 1.0) return false; - if (c < 0.0 || c > 1.0) return false; - - return true; - } - -private: - size_t _offset_x = 0; - size_t _offset_y = 0; - - aliceVision::image::Image _coordinates; - aliceVision::image::Image _mask; -}; - -class AlphaBuilder { -public: - virtual bool build(const CoordinatesMap & map, const aliceVision::camera::IntrinsicBase & intrinsics) { - - float w = static_cast(intrinsics.w()); - float h = static_cast(intrinsics.h()); - float cx = w / 2.0f; - float cy = h / 2.0f; - - - const aliceVision::image::Image & coordinates = map.getCoordinates(); - const aliceVision::image::Image & mask = map.getMask(); - - _weights = aliceVision::image::Image(coordinates.Width(), coordinates.Height()); - - for (int i = 0; i < _weights.Height(); i++) { - for (int j = 0; j < _weights.Width(); j++) { - - _weights(i, j) = 0.0f; - - bool valid = mask(i, j); - if (!valid) { - continue; - } - - const Vec2 & coords = coordinates(i, j); - - float x = coords(0); - float y = coords(1); - - float wx = 1.0f - std::abs((x - cx) / cx); - float wy = 1.0f - std::abs((y - cy) / cy); - - _weights(i, j) = wx * wy; - } - } - - return true; - } - - const aliceVision::image::Image & getWeights() const { - return _weights; - } - -private: - aliceVision::image::Image _weights; -}; - -class Warper { -public: - virtual bool warp(const CoordinatesMap & map, const aliceVision::image::Image & source) { - - /** - * Copy additional info from map - */ - _offset_x = map.getOffsetX(); - _offset_y = map.getOffsetY(); - _mask = map.getMask(); - - const image::Sampler2d sampler; - const aliceVision::image::Image & coordinates = map.getCoordinates(); - - /** - * Create buffer - * No longer need to keep a 2**x size - */ - _color = aliceVision::image::Image(coordinates.Width(), coordinates.Height()); - - /** - * Simple warp - */ - for (size_t i = 0; i < _color.Height(); i++) { - for (size_t j = 0; j < _color.Width(); j++) { - - bool valid = _mask(i, j); - if (!valid) { - continue; - } - - const Eigen::Vector2d & coord = coordinates(i, j); - const image::RGBfColor pixel = sampler(source, coord(1), coord(0)); - - _color(i, j) = pixel; - } - } - - return true; - } - - const aliceVision::image::Image & getColor() const { - return _color; - } - - const aliceVision::image::Image & getMask() const { - return _mask; - } - - - size_t getOffsetX() const { - return _offset_x; - } - - size_t getOffsetY() const { - return _offset_y; - } - -protected: - size_t _offset_x = 0; - size_t _offset_y = 0; - - aliceVision::image::Image _color; - aliceVision::image::Image _mask; -}; - -class GaussianWarper : public Warper { -public: - virtual bool warp(const CoordinatesMap & map, const aliceVision::image::Image & source) { - - /** - * Copy additional info from map - */ - _offset_x = map.getOffsetX(); - _offset_y = map.getOffsetY(); - _mask = map.getMask(); - - const image::Sampler2d sampler; - const aliceVision::image::Image & coordinates = map.getCoordinates(); - - /** - * Create a pyramid for input - */ - GaussianPyramidNoMask pyramid(source.Width(), source.Height()); - pyramid.process(source); - const std::vector> & mlsource = pyramid.getPyramidColor(); - size_t max_level = pyramid.getScalesCount() - 1; - - /** - * Create buffer - */ - _color = aliceVision::image::Image(coordinates.Width(), coordinates.Height(), true, image::RGBfColor(1.0, 0.0, 0.0)); - - - /** - * Multi level warp - */ - for (size_t i = 0; i < _color.Height(); i++) { - for (size_t j = 0; j < _color.Width(); j++) { - - bool valid = _mask(i, j); - if (!valid) { - continue; - } - - if (i == _color.Height() - 1 || j == _color.Width() - 1 || !_mask(i + 1, j) || !_mask(i, j + 1)) { - const Eigen::Vector2d & coord = coordinates(i, j); - const image::RGBfColor pixel = sampler(source, coord(1), coord(0)); - _color(i, j) = pixel; - continue; - } - - const Eigen::Vector2d & coord_mm = coordinates(i, j); - const Eigen::Vector2d & coord_mp = coordinates(i, j + 1); - const Eigen::Vector2d & coord_pm = coordinates(i + 1, j); - - double dxx = coord_pm(0) - coord_mm(0); - double dxy = coord_mp(0) - coord_mm(0); - double dyx = coord_pm(1) - coord_mm(1); - double dyy = coord_mp(1) - coord_mm(1); - double det = std::abs(dxx*dyy - dxy*dyx); - double scale = sqrt(det); - - double flevel = std::max(0.0, log2(scale)); - size_t blevel = std::min(max_level, size_t(floor(flevel))); - - double dscale, x, y; - dscale = 1.0 / pow(2.0, blevel); - x = coord_mm(0) * dscale; - y = coord_mm(1) * dscale; - /*Fallback to first level if outside*/ - if (x >= mlsource[blevel].Width() - 1 || y >= mlsource[blevel].Height() - 1) { - _color(i, j) = sampler(mlsource[0], coord_mm(1), coord_mm(0)); - continue; - } - - _color(i, j) = sampler(mlsource[blevel], y, x); - } - } - - return true; - } -}; - -bool computeOptimalPanoramaSize(std::pair & optimalSize, const sfmData::SfMData & sfmData, const float ratioUpscale) { - - optimalSize.first = 512; - optimalSize.second = 256; - - /** - * Loop over views to estimate best scale - */ - std::vector scales; - for (auto & viewIt: sfmData.getViews()) { - - /** - * Retrieve view - */ - const sfmData::View& view = *viewIt.second.get(); - if (!sfmData.isPoseAndIntrinsicDefined(&view)) { - continue; - } - - /** - * Get intrinsics and extrinsics - */ - const geometry::Pose3 camPose = sfmData.getPose(view).getTransform(); - const camera::IntrinsicBase & intrinsic = *sfmData.getIntrinsicPtr(view.getIntrinsicId()); - - /** - * Compute map - */ - CoordinatesMap map; - if (!map.build(optimalSize, camPose, intrinsic)) { - continue; - } - - double scale; - if (!map.computeScale(scale, ratioUpscale)) { - continue; - } - - scales.push_back(scale); - } - - - if (scales.empty()) { - return false; - } - - std::sort(scales.begin(), scales.end()); - const float indexRatio = float(scales.size() - 1) * ratioUpscale; - float intPart = 0.0f; - const float fractPart = std::modf(indexRatio, &intPart); - const size_t indexL = int(intPart); - const size_t indexU = std::min(scales.size() - 1, size_t(std::ceil(indexRatio))); - const double selectedScale = scales[indexL] * (1.0f - fractPart) + scales[indexU] * fractPart; - - optimalSize.first = optimalSize.first * selectedScale; - optimalSize.second = optimalSize.second * selectedScale; - - ALICEVISION_LOG_INFO("Estimated panorama size: " << optimalSize.first << "x" << optimalSize.second); - - return true; + // We use a small panorama for probing + optimalSize.first = 512; + optimalSize.second = 256; + + // Loop over views to estimate best scale + std::vector scales; + for(auto& viewIt : sfmData.getViews()) + { + + // Ignore non positionned views + const sfmData::View& view = *viewIt.second.get(); + if(!sfmData.isPoseAndIntrinsicDefined(&view)) + { + continue; + } + + // Get intrinsics and extrinsics + const geometry::Pose3 camPose = sfmData.getPose(view).getTransform(); + const camera::IntrinsicBase& intrinsic = *sfmData.getIntrinsicPtr(view.getIntrinsicId()); + + // Compute coarse bounding box + BoundingBox coarseBbox; + if(!computeCoarseBB(coarseBbox, optimalSize, camPose, intrinsic)) + { + continue; + } + + CoordinatesMap map; + if(!map.build(optimalSize, camPose, intrinsic, coarseBbox)) + { + continue; + } + + double scale; + if(!map.computeScale(scale, ratioUpscale)) + { + continue; + } + + scales.push_back(scale); + } + + if(scales.empty()) + { + return false; + } + + std::sort(scales.begin(), scales.end()); + int selected_index = int(floor(float(scales.size() - 1) * ratioUpscale)); + double selected_scale = scales[selected_index]; + + optimalSize.first = optimalSize.first * selected_scale; + optimalSize.second = optimalSize.second * selected_scale; + + ALICEVISION_LOG_INFO("Estimated panorama size: " << optimalSize.first << "x" << optimalSize.second); + + return true; } -int aliceVision_main(int argc, char **argv) +int aliceVision_main(int argc, char** argv) { - std::string sfmDataFilename; - std::string outputDirectory; - - std::pair panoramaSize = {0, 0}; - int percentUpscale = 50; - - image::EStorageDataType storageDataType = image::EStorageDataType::Float; - - int rangeStart = -1; - int rangeSize = 1; - - // Program description - po::options_description allParams ( - "Perform panorama stiching of cameras around a nodal point for 360° panorama creation. \n" - "AliceVision PanoramaWarping" - ); - - // Description of mandatory parameters - po::options_description requiredParams("Required parameters"); - requiredParams.add_options() - ("input,i", po::value(&sfmDataFilename)->required(), "SfMData file.") - ("output,o", po::value(&outputDirectory)->required(), "Path of the output folder."); - allParams.add(requiredParams); - - // Description of optional parameters - po::options_description optionalParams("Optional parameters"); - optionalParams.add_options() - ("panoramaWidth,w", po::value(&panoramaSize.first)->default_value(panoramaSize.first), - "Panorama Width in pixels.") - ("percentUpscale", po::value(&percentUpscale)->default_value(percentUpscale), - "Percentage of upscaled pixels.") - ("storageDataType", po::value(&storageDataType)->default_value(storageDataType), - ("Storage data type: " + image::EStorageDataType_informations()).c_str()) - ("rangeStart", po::value(&rangeStart)->default_value(rangeStart), - "Range image index start.") - ("rangeSize", po::value(&rangeSize)->default_value(rangeSize), - "Range size."); - allParams.add(optionalParams); - - // Setup log level given command line - std::string verboseLevel = system::EVerboseLevel_enumToString(system::Logger::getDefaultVerboseLevel()); - po::options_description logParams("Log parameters"); - logParams.add_options() - ("verboseLevel,v", po::value(&verboseLevel)->default_value(verboseLevel), "verbosity level (fatal, error, warning, info, debug, trace)."); - allParams.add(logParams); - - // Effectively parse command line given parse options - po::variables_map vm; - try - { - po::store(po::parse_command_line(argc, argv, allParams), vm); - - if(vm.count("help") || (argc == 1)) - { - ALICEVISION_COUT(allParams); - return EXIT_SUCCESS; - } - po::notify(vm); - } - catch(boost::program_options::required_option& e) - { - ALICEVISION_CERR("ERROR: " << e.what()); - ALICEVISION_COUT("Usage:\n\n" << allParams); - return EXIT_FAILURE; - } - catch(boost::program_options::error& e) - { - ALICEVISION_CERR("ERROR: " << e.what()); - ALICEVISION_COUT("Usage:\n\n" << allParams); - return EXIT_FAILURE; - } - - ALICEVISION_COUT("Program called with the following parameters:"); - ALICEVISION_COUT(vm); - - // Set verbose level given command line - system::Logger::get()->setLogLevel(verboseLevel); - - // Load information about inputs - // Camera images - // Camera intrinsics - // Camera extrinsics - sfmData::SfMData sfmData; - if(!sfmDataIO::Load(sfmData, sfmDataFilename, sfmDataIO::ESfMData(sfmDataIO::VIEWS| sfmDataIO::INTRINSICS| sfmDataIO::EXTRINSICS))) - { - ALICEVISION_LOG_ERROR("The input SfMData file '" << sfmDataFilename << "' cannot be read."); - return EXIT_FAILURE; - } - - // Order views by their image names for easier debugging - std::vector> viewsOrderedByName; - for (auto & viewIt: sfmData.getViews()) { - viewsOrderedByName.push_back(viewIt.second); - } - std::sort(viewsOrderedByName.begin(), viewsOrderedByName.end(), [](const std::shared_ptr & a, const std::shared_ptr & b) -> bool { - if (a == nullptr || b == nullptr) return true; - return (a->getImagePath() < b->getImagePath()); - }); - - - // If panorama width is undefined, estimate it - if (panoramaSize.first <= 0) - { - float ratioUpscale = clamp(float(percentUpscale) / 100.0f, 0.0f, 1.0f); - - - std::pair optimalPanoramaSize; - if (computeOptimalPanoramaSize(optimalPanoramaSize, sfmData, ratioUpscale)) - { - panoramaSize = optimalPanoramaSize; - } - else { - ALICEVISION_LOG_INFO("Impossible to compute an optimal panorama size"); - return EXIT_FAILURE; - } - } - - double max_scale = 1.0 / pow(2.0, 10); - panoramaSize.first = int(ceil(double(panoramaSize.first) * max_scale) / max_scale); - panoramaSize.second = panoramaSize.first / 2; - - ALICEVISION_LOG_INFO("Choosen panorama size : " << panoramaSize.first << "x" << panoramaSize.second); - - // Define range to compute - if(rangeStart != -1) - { - if(rangeStart < 0 || rangeSize < 0 || - std::size_t(rangeStart) > viewsOrderedByName.size()) - { - ALICEVISION_LOG_ERROR("Range is incorrect"); - return EXIT_FAILURE; - } - - if(std::size_t(rangeStart + rangeSize) > viewsOrderedByName.size()) - { - rangeSize = int(viewsOrderedByName.size()) - rangeStart; - } - } - else - { - rangeStart = 0; - rangeSize = int(viewsOrderedByName.size()); - } - ALICEVISION_LOG_DEBUG("Range to compute: rangeStart=" << rangeStart << ", rangeSize=" << rangeSize); - - // Preprocessing per view - for(std::size_t i = std::size_t(rangeStart); i < std::size_t(rangeStart + rangeSize); ++i) - { - const std::shared_ptr & viewIt = viewsOrderedByName[i]; - - // Retrieve view - const sfmData::View& view = *viewIt; - if (!sfmData.isPoseAndIntrinsicDefined(&view)) - { - continue; - } - - ALICEVISION_LOG_INFO("[" << int(i) + 1 - rangeStart << "/" << rangeSize << "] Processing view " << view.getViewId() << " (" << i + 1 << "/" << viewsOrderedByName.size() << ")"); - - // Get intrinsics and extrinsics - geometry::Pose3 camPose = sfmData.getPose(view).getTransform(); - std::shared_ptr intrinsic = sfmData.getIntrinsicsharedPtr(view.getIntrinsicId()); - std::shared_ptr casted = std::dynamic_pointer_cast(intrinsic); - - // Prepare coordinates map - CoordinatesMap map; - map.build(panoramaSize, camPose, *(intrinsic.get())); - - // Load image and convert it to linear colorspace - std::string imagePath = view.getImagePath(); - ALICEVISION_LOG_INFO("Load image with path " << imagePath); - image::Image source; - image::readImage(imagePath, source, image::EImageColorSpace::LINEAR); - - // Warp image - GaussianWarper warper; - warper.warp(map, source); - - // Alpha mask - AlphaBuilder alphabuilder; - alphabuilder.build(map, *(intrinsic.get())); - - // Export mask and image - { - const std::string viewIdStr = std::to_string(view.getViewId()); - - oiio::ParamValueList metadata = image::readImageMetadata(imagePath); - const int offsetX = int(warper.getOffsetX()); - const int offsetY = int(warper.getOffsetY()); - metadata.push_back(oiio::ParamValue("AliceVision:offsetX", offsetX)); - metadata.push_back(oiio::ParamValue("AliceVision:offsetY", offsetY)); - metadata.push_back(oiio::ParamValue("AliceVision:panoramaWidth", panoramaSize.first)); - metadata.push_back(oiio::ParamValue("AliceVision:panoramaHeight", panoramaSize.second)); - - // Images will be converted in Panorama coordinate system, so there will be no more extra orientation. - metadata.remove("Orientation"); - metadata.remove("orientation"); - - { - const aliceVision::image::Image & cam = warper.getColor(); - - oiio::ParamValueList viewMetadata = metadata; - viewMetadata.push_back(oiio::ParamValue("AliceVision:storageDataType", image::EStorageDataType_enumToString(storageDataType))); - - const std::string viewFilepath = (fs::path(outputDirectory) / (viewIdStr + ".exr")).string(); - ALICEVISION_LOG_INFO("Store view " << i << " with path " << viewFilepath); - image::writeImage(viewFilepath, cam, image::EImageColorSpace::AUTO, viewMetadata); - } - { - const aliceVision::image::Image & mask = warper.getMask(); - - const std::string maskFilepath = (fs::path(outputDirectory) / (viewIdStr + "_mask.exr")).string(); - ALICEVISION_LOG_INFO("Store mask " << i << " with path " << maskFilepath); - image::writeImage(maskFilepath, mask, image::EImageColorSpace::NO_CONVERSION, metadata); - } - { - const aliceVision::image::Image & weights = alphabuilder.getWeights(); - - const std::string weightFilepath = (fs::path(outputDirectory) / (viewIdStr + "_weight.exr")).string(); - ALICEVISION_LOG_INFO("Store weightmap " << i << " with path " << weightFilepath); - image::writeImage(weightFilepath, weights, image::EImageColorSpace::AUTO, metadata); - } - } - } - - return EXIT_SUCCESS; + std::string sfmDataFilename; + std::string outputDirectory; + std::pair panoramaSize = {0, 0}; + int percentUpscale = 50; + int tileSize = 256; + + image::EStorageDataType storageDataType = image::EStorageDataType::Float; + + int rangeStart = -1; + int rangeSize = 1; + + // Program description + po::options_description allParams( + "Perform panorama stiching of cameras around a nodal point for 360° panorama creation. \n" + "AliceVision PanoramaWarping"); + + // Description of mandatory parameters + po::options_description requiredParams("Required parameters"); + requiredParams.add_options()("input,i", po::value(&sfmDataFilename)->required(), "SfMData file.")( + "output,o", po::value(&outputDirectory)->required(), "Path of the output folder."); + allParams.add(requiredParams); + + // Description of optional parameters + po::options_description optionalParams("Optional parameters"); + optionalParams.add_options()( + "panoramaWidth,w", po::value(&panoramaSize.first)->default_value(panoramaSize.first), + "Panorama Width in pixels.")("percentUpscale", po::value(&percentUpscale)->default_value(percentUpscale), + "Percentage of upscaled pixels.")( + "storageDataType", po::value(&storageDataType)->default_value(storageDataType), + ("Storage data type: " + image::EStorageDataType_informations()).c_str())( + "rangeStart", po::value(&rangeStart)->default_value(rangeStart), + "Range image index start.")("rangeSize", po::value(&rangeSize)->default_value(rangeSize), "Range size."); + allParams.add(optionalParams); + + // Setup log level given command line + std::string verboseLevel = system::EVerboseLevel_enumToString(system::Logger::getDefaultVerboseLevel()); + po::options_description logParams("Log parameters"); + logParams.add_options()("verboseLevel,v", po::value(&verboseLevel)->default_value(verboseLevel), + "verbosity level (fatal, error, warning, info, debug, trace)."); + allParams.add(logParams); + + // Effectively parse command line given parse options + po::variables_map vm; + try + { + po::store(po::parse_command_line(argc, argv, allParams), vm); + + if(vm.count("help") || (argc == 1)) + { + ALICEVISION_COUT(allParams); + return EXIT_SUCCESS; + } + po::notify(vm); + } + catch(boost::program_options::required_option& e) + { + ALICEVISION_CERR("ERROR: " << e.what()); + ALICEVISION_COUT("Usage:\n\n" << allParams); + return EXIT_FAILURE; + } + catch(boost::program_options::error& e) + { + ALICEVISION_CERR("ERROR: " << e.what()); + ALICEVISION_COUT("Usage:\n\n" << allParams); + return EXIT_FAILURE; + } + + ALICEVISION_COUT("Program called with the following parameters:"); + ALICEVISION_COUT(vm); + + // Set verbose level given command line + system::Logger::get()->setLogLevel(verboseLevel); + + // Load information about inputs + // Camera images + // Camera intrinsics + // Camera extrinsics + sfmData::SfMData sfmData; + if(!sfmDataIO::Load(sfmData, sfmDataFilename, + sfmDataIO::ESfMData(sfmDataIO::VIEWS | sfmDataIO::INTRINSICS | sfmDataIO::EXTRINSICS))) + { + ALICEVISION_LOG_ERROR("The input SfMData file '" << sfmDataFilename << "' cannot be read."); + return EXIT_FAILURE; + } + + // Order views by their image names for easier debugging + std::vector> viewsOrderedByName; + for(auto& viewIt : sfmData.getViews()) + { + viewsOrderedByName.push_back(viewIt.second); + } + std::sort(viewsOrderedByName.begin(), viewsOrderedByName.end(), + [](const std::shared_ptr& a, const std::shared_ptr& b) -> bool { + if(a == nullptr || b == nullptr) + return true; + return (a->getImagePath() < b->getImagePath()); + }); + + + // Define range to compute + if(rangeStart != -1) + { + if(rangeStart < 0 || rangeSize < 0 || + std::size_t(rangeStart) > viewsOrderedByName.size()) + { + ALICEVISION_LOG_ERROR("Range is incorrect"); + return EXIT_FAILURE; + } + + if(std::size_t(rangeStart + rangeSize) > viewsOrderedByName.size()) + { + rangeSize = int(viewsOrderedByName.size()) - rangeStart; + } + } + else + { + rangeStart = 0; + rangeSize = int(viewsOrderedByName.size()); + } + ALICEVISION_LOG_DEBUG("Range to compute: rangeStart=" << rangeStart << ", rangeSize=" << rangeSize); + + + // If panorama width is undefined, estimate it + if(panoramaSize.first <= 0) + { + float ratioUpscale = clamp(float(percentUpscale) / 100.0f, 0.0f, 1.0f); + + std::pair optimalPanoramaSize; + if(computeOptimalPanoramaSize(optimalPanoramaSize, sfmData, ratioUpscale)) + { + panoramaSize = optimalPanoramaSize; + } + else + { + ALICEVISION_LOG_INFO("Impossible to compute an optimal panorama size"); + return EXIT_FAILURE; + } + } + + // Make sure panorama size has required size properties + double max_scale = 1.0 / pow(2.0, 10); + panoramaSize.first = int(ceil(double(panoramaSize.first) * max_scale) / max_scale); + panoramaSize.second = panoramaSize.first / 2; + ALICEVISION_LOG_INFO("Choosen panorama size : " << panoramaSize.first << "x" << panoramaSize.second); + + + // Define empty tiles data + std::unique_ptr empty_float(new float[tileSize * tileSize * 3]); + std::unique_ptr empty_char(new char[tileSize * tileSize]); + std::memset(empty_float.get(), 0, tileSize * tileSize * 3 * sizeof(float)); + std::memset(empty_char.get(), 0, tileSize * tileSize * sizeof(char)); + + // Preprocessing per view + for(std::size_t i = std::size_t(rangeStart); i < std::size_t(rangeStart + rangeSize); ++i) + { + const std::shared_ptr & viewIt = viewsOrderedByName[i]; + + // Retrieve view + const sfmData::View& view = *viewIt; + if (!sfmData.isPoseAndIntrinsicDefined(&view)) + { + continue; + } + + ALICEVISION_LOG_INFO("[" << int(i) + 1 - rangeStart << "/" << rangeSize << "] Processing view " << view.getViewId() << " (" << i + 1 << "/" << viewsOrderedByName.size() << ")"); + + // Get intrinsics and extrinsics + geometry::Pose3 camPose = sfmData.getPose(view).getTransform(); + std::shared_ptr intrinsic = sfmData.getIntrinsicsharedPtr(view.getIntrinsicId()); + + // Compute coarse bounding box to make computations faster + BoundingBox coarseBbox; + if (!computeCoarseBB(coarseBbox, panoramaSize, camPose, *(intrinsic.get()))) + { + continue; + } + + // round to the closest tiles + coarseBbox.snapToGrid(tileSize); + + //Initialize bouding box for image + BoundingBox globalBbox; + + #pragma omp parallel for + for (int y = 0; y < coarseBbox.height; y += tileSize) { + for (int x = 0; x < coarseBbox.width; x += tileSize) { + + BoundingBox localBbox; + localBbox.left = x + coarseBbox.left; + localBbox.top = y + coarseBbox.top; + localBbox.width = tileSize; + localBbox.height = tileSize; + + // Prepare coordinates map + CoordinatesMap map; + if (!map.build(panoramaSize, camPose, *(intrinsic.get()), localBbox)) { + continue; + } + + #pragma omp critical + { + globalBbox.unionWith(map.getBoundingBox()); + } + } + } + + + // Update bounding box + coarseBbox = globalBbox; + coarseBbox.snapToGrid(tileSize); + if (coarseBbox.width <= 0 || coarseBbox.height <= 0) continue; + + + // Load image and convert it to linear colorspace + std::string imagePath = view.getImagePath(); + ALICEVISION_LOG_INFO("Load image with path " << imagePath); + image::Image source; + image::readImage(imagePath, source, image::EImageColorSpace::LINEAR); + + // Load metadata and update for output + oiio::ParamValueList metadata = image::readImageMetadata(imagePath); + metadata.push_back(oiio::ParamValue("AliceVision:offsetX", coarseBbox.left)); + metadata.push_back(oiio::ParamValue("AliceVision:offsetY", coarseBbox.top)); + metadata.push_back(oiio::ParamValue("AliceVision:contentX", globalBbox.left - coarseBbox.left)); + metadata.push_back(oiio::ParamValue("AliceVision:contentY", globalBbox.top - coarseBbox.top)); + metadata.push_back(oiio::ParamValue("AliceVision:contentW", globalBbox.width)); + metadata.push_back(oiio::ParamValue("AliceVision:contentH", globalBbox.height)); + metadata.push_back(oiio::ParamValue("AliceVision:panoramaWidth", panoramaSize.first)); + metadata.push_back(oiio::ParamValue("AliceVision:panoramaHeight", panoramaSize.second)); + + // Images will be converted in Panorama coordinate system, so there will be no more extra orientation. + metadata.remove("Orientation"); + metadata.remove("orientation"); + + // Define output paths + const std::string viewIdStr = std::to_string(view.getViewId()); + const std::string viewFilepath = (fs::path(outputDirectory) / (viewIdStr + ".exr")).string(); + const std::string maskFilepath = (fs::path(outputDirectory) / (viewIdStr + "_mask.exr")).string(); + const std::string weightFilepath = (fs::path(outputDirectory) / (viewIdStr + "_weight.exr")).string(); + + // Create output images + std::unique_ptr out_view = oiio::ImageOutput::create(viewFilepath); + std::unique_ptr out_mask = oiio::ImageOutput::create(maskFilepath); + std::unique_ptr out_weights = oiio::ImageOutput::create(weightFilepath); + + // Define output properties + oiio::ImageSpec spec_view(coarseBbox.width, coarseBbox.height, 3, (storageDataType == image::EStorageDataType::Half)?oiio::TypeDesc::HALF:oiio::TypeDesc::FLOAT); + oiio::ImageSpec spec_mask(coarseBbox.width, coarseBbox.height, 1, oiio::TypeDesc::UCHAR); + oiio::ImageSpec spec_weights(coarseBbox.width, coarseBbox.height, 1, oiio::TypeDesc::HALF); + + spec_view.tile_width = tileSize; + spec_view.tile_height = tileSize; + spec_mask.tile_width = tileSize; + spec_mask.tile_height = tileSize; + spec_weights.tile_width = tileSize; + spec_weights.tile_height = tileSize; + spec_view.attribute("compression", "piz"); + spec_weights.attribute("compression", "piz"); + spec_mask.attribute("compression", "piz"); + spec_view.extra_attribs = metadata; + spec_mask.extra_attribs = metadata; + spec_weights.extra_attribs = metadata; + + out_view->open(viewFilepath, spec_view); + out_mask->open(maskFilepath, spec_mask); + out_weights->open(weightFilepath, spec_weights); + + GaussianPyramidNoMask pyramid(source.Width(), source.Height()); + if (!pyramid.process(source)) { + ALICEVISION_LOG_ERROR("Problem creating pyramid."); + continue; + } + + #pragma omp parallel for + { + for (int y = 0; y < coarseBbox.height; y += tileSize) + { + for (int x = 0; x < coarseBbox.width; x += tileSize) + { + BoundingBox localBbox; + localBbox.left = x + coarseBbox.left; + localBbox.top = y + coarseBbox.top; + localBbox.width = tileSize; + localBbox.height = tileSize; + + // Prepare coordinates map + CoordinatesMap map; + if (!map.build(panoramaSize, camPose, *(intrinsic.get()), localBbox)) + { + continue; + } + + // Warp image + GaussianWarper warper; + if (!warper.warp(map, pyramid)) { + continue; + } + + // Alpha mask + aliceVision::image::Image weights; + if (!distanceToCenter(weights, map, intrinsic->w(), intrinsic->h())) { + continue; + } + + // Store + #pragma omp critical + { + out_view->write_tile(x, y, 0, oiio::TypeDesc::FLOAT, warper.getColor().data()); + out_mask->write_tile(x, y, 0, oiio::TypeDesc::UCHAR, warper.getMask().data()); + out_weights->write_tile(x, y, 0, oiio::TypeDesc::FLOAT, weights.data()); + } + } + } + } + + out_view->close(); + out_mask->close(); + out_weights->close(); + } + + return EXIT_SUCCESS; } From bc1672c6e02aea9550571bb887b00c68bf9bf8c9 Mon Sep 17 00:00:00 2001 From: Fabien Servant Date: Thu, 1 Oct 2020 15:21:41 +0200 Subject: [PATCH 03/79] extract everything from compositing and put in modules --- src/aliceVision/panorama/CMakeLists.txt | 4 + src/aliceVision/panorama/alphaCompositer.hpp | 85 + src/aliceVision/panorama/boundingBox.hpp | 34 +- src/aliceVision/panorama/compositer.hpp | 62 + src/aliceVision/panorama/coordinatesMap.cpp | 162 +- src/aliceVision/panorama/coordinatesMap.hpp | 76 +- src/aliceVision/panorama/distance.cpp | 118 +- src/aliceVision/panorama/distance.hpp | 8 +- src/aliceVision/panorama/feathering.cpp | 126 + src/aliceVision/panorama/feathering.hpp | 12 + src/aliceVision/panorama/gaussian.cpp | 62 +- src/aliceVision/panorama/gaussian.hpp | 177 +- src/aliceVision/panorama/graphcut.hpp | 872 +++++++ src/aliceVision/panorama/imageOps.cpp | 36 + src/aliceVision/panorama/imageOps.hpp | 116 + .../panorama/laplacianCompositer.hpp | 155 ++ src/aliceVision/panorama/laplacianPyramid.cpp | 356 +++ src/aliceVision/panorama/laplacianPyramid.hpp | 25 + src/aliceVision/panorama/remapBbox.cpp | 536 +++-- src/aliceVision/panorama/remapBbox.hpp | 8 +- src/aliceVision/panorama/seams.cpp | 320 +++ src/aliceVision/panorama/seams.hpp | 78 + src/aliceVision/panorama/sphericalMapping.cpp | 19 +- src/aliceVision/panorama/sphericalMapping.hpp | 12 +- src/aliceVision/panorama/warper.cpp | 134 +- src/aliceVision/panorama/warper.hpp | 43 +- src/software/pipeline/CMakeLists.txt | 1 + .../pipeline/main_panoramaCompositing.cpp | 2122 +---------------- .../pipeline/main_panoramaWarping.cpp | 128 +- 29 files changed, 3164 insertions(+), 2723 deletions(-) create mode 100644 src/aliceVision/panorama/alphaCompositer.hpp create mode 100644 src/aliceVision/panorama/compositer.hpp create mode 100644 src/aliceVision/panorama/feathering.cpp create mode 100644 src/aliceVision/panorama/feathering.hpp create mode 100644 src/aliceVision/panorama/graphcut.hpp create mode 100644 src/aliceVision/panorama/imageOps.cpp create mode 100644 src/aliceVision/panorama/imageOps.hpp create mode 100644 src/aliceVision/panorama/laplacianCompositer.hpp create mode 100644 src/aliceVision/panorama/laplacianPyramid.cpp create mode 100644 src/aliceVision/panorama/laplacianPyramid.hpp create mode 100644 src/aliceVision/panorama/seams.cpp create mode 100644 src/aliceVision/panorama/seams.hpp diff --git a/src/aliceVision/panorama/CMakeLists.txt b/src/aliceVision/panorama/CMakeLists.txt index 915fdc9391..dfa5123554 100644 --- a/src/aliceVision/panorama/CMakeLists.txt +++ b/src/aliceVision/panorama/CMakeLists.txt @@ -12,6 +12,10 @@ set(panorama_files_sources distance.cpp remapBbox.cpp sphericalMapping.cpp + feathering.cpp + laplacianPyramid.cpp + seams.cpp + imageOps.cpp ) alicevision_add_library(aliceVision_panorama diff --git a/src/aliceVision/panorama/alphaCompositer.hpp b/src/aliceVision/panorama/alphaCompositer.hpp new file mode 100644 index 0000000000..6bb3999ee1 --- /dev/null +++ b/src/aliceVision/panorama/alphaCompositer.hpp @@ -0,0 +1,85 @@ +#pragma once + +#include "compositer.hpp" + +namespace aliceVision +{ + +class AlphaCompositer : public Compositer +{ +public: + AlphaCompositer(size_t outputWidth, size_t outputHeight) + : Compositer(outputWidth, outputHeight) + { + } + + virtual bool append(const aliceVision::image::Image& color, + const aliceVision::image::Image& inputMask, + const aliceVision::image::Image& inputWeights, size_t offset_x, size_t offset_y) + { + + for(size_t i = 0; i < color.Height(); i++) + { + + size_t pano_i = offset_y + i; + if(pano_i >= _panorama.Height()) + { + continue; + } + + for(size_t j = 0; j < color.Width(); j++) + { + + if(!inputMask(i, j)) + { + continue; + } + + size_t pano_j = offset_x + j; + if(pano_j >= _panorama.Width()) + { + pano_j = pano_j - _panorama.Width(); + } + + float wc = inputWeights(i, j); + + _panorama(pano_i, pano_j).r() += wc * color(i, j).r(); + _panorama(pano_i, pano_j).g() += wc * color(i, j).g(); + _panorama(pano_i, pano_j).b() += wc * color(i, j).b(); + _panorama(pano_i, pano_j).a() += wc; + } + } + + return true; + } + + virtual bool terminate() + { + + for(int i = 0; i < _panorama.Height(); i++) + { + for(int j = 0; j < _panorama.Width(); j++) + { + + if(_panorama(i, j).a() < 1e-6) + { + _panorama(i, j).r() = 1.0f; + _panorama(i, j).g() = 0.0f; + _panorama(i, j).b() = 0.0f; + _panorama(i, j).a() = 0.0f; + } + else + { + _panorama(i, j).r() = _panorama(i, j).r() / _panorama(i, j).a(); + _panorama(i, j).g() = _panorama(i, j).g() / _panorama(i, j).a(); + _panorama(i, j).b() = _panorama(i, j).b() / _panorama(i, j).a(); + _panorama(i, j).a() = 1.0f; + } + } + } + + return true; + } +}; + +} // namespace aliceVision diff --git a/src/aliceVision/panorama/boundingBox.hpp b/src/aliceVision/panorama/boundingBox.hpp index 780e3b7dde..5883c4f01c 100644 --- a/src/aliceVision/panorama/boundingBox.hpp +++ b/src/aliceVision/panorama/boundingBox.hpp @@ -1,35 +1,39 @@ #pragma once - -struct BoundingBox { +struct BoundingBox +{ int left; int top; int width; int height; - BoundingBox() { + BoundingBox() + { left = -1; top = -1; width = 0; height = 0; }; - BoundingBox(int l, int t, int w, int h) : left(l), top(t), width(w), height(h) {} - - int getRight() const { - return left + width - 1; + BoundingBox(int l, int t, int w, int h) + : left(l) + , top(t) + , width(w) + , height(h) + { } - int getBottom() const { - return top + height - 1; - } + int getRight() const { return left + width - 1; } - void snapToGrid(uint32_t gridSize) { + int getBottom() const { return top + height - 1; } + + void snapToGrid(uint32_t gridSize) + { int right = getRight(); int bottom = getBottom(); - + int leftBounded = int(floor(double(left) / double(gridSize))) * int(gridSize); int topBounded = int(floor(double(top) / double(gridSize))) * int(gridSize); int widthBounded = int(ceil(double(right - leftBounded + 1) / double(gridSize))) * int(gridSize); @@ -41,9 +45,11 @@ struct BoundingBox { height = heightBounded; } - void unionWith(const BoundingBox & other) { + void unionWith(const BoundingBox& other) + { - if (left < 0 && top < 0) { + if(left < 0 && top < 0) + { left = other.left; top = other.top; width = other.width; diff --git a/src/aliceVision/panorama/compositer.hpp b/src/aliceVision/panorama/compositer.hpp new file mode 100644 index 0000000000..c7aeaef010 --- /dev/null +++ b/src/aliceVision/panorama/compositer.hpp @@ -0,0 +1,62 @@ +#pragma once + +#include + +namespace aliceVision +{ + +class Compositer +{ +public: + Compositer(size_t outputWidth, size_t outputHeight) + : _panorama(outputWidth, outputHeight, true, image::RGBAfColor(0.0f, 0.0f, 0.0f, 0.0f)) + { + } + + virtual bool append(const aliceVision::image::Image& color, + const aliceVision::image::Image& inputMask, + const aliceVision::image::Image& inputWeights, size_t offset_x, size_t offset_y) + { + + for(size_t i = 0; i < color.Height(); i++) + { + + size_t pano_i = offset_y + i; + if(pano_i >= _panorama.Height()) + { + continue; + } + + for(size_t j = 0; j < color.Width(); j++) + { + + if(!inputMask(i, j)) + { + continue; + } + + size_t pano_j = offset_x + j; + if(pano_j >= _panorama.Width()) + { + pano_j = pano_j - _panorama.Width(); + } + + _panorama(pano_i, pano_j).r() = color(i, j).r(); + _panorama(pano_i, pano_j).g() = color(i, j).g(); + _panorama(pano_i, pano_j).b() = color(i, j).b(); + _panorama(pano_i, pano_j).a() = 1.0f; + } + } + + return true; + } + + virtual bool terminate() { return true; } + + aliceVision::image::Image& getPanorama() { return _panorama; } + +protected: + aliceVision::image::Image _panorama; +}; + +} // namespace aliceVision \ No newline at end of file diff --git a/src/aliceVision/panorama/coordinatesMap.cpp b/src/aliceVision/panorama/coordinatesMap.cpp index 9cd3b00d98..d20df974f0 100644 --- a/src/aliceVision/panorama/coordinatesMap.cpp +++ b/src/aliceVision/panorama/coordinatesMap.cpp @@ -4,11 +4,13 @@ #include "sphericalMapping.hpp" -namespace aliceVision { +namespace aliceVision +{ +bool CoordinatesMap::build(const std::pair& panoramaSize, const geometry::Pose3& pose, + const aliceVision::camera::IntrinsicBase& intrinsics, const BoundingBox& coarseBbox) +{ -bool CoordinatesMap::build(const std::pair & panoramaSize, const geometry::Pose3 & pose, const aliceVision::camera::IntrinsicBase & intrinsics, const BoundingBox &coarseBbox) { - /* Effectively compute the warping map */ _coordinates = aliceVision::image::Image(coarseBbox.width, coarseBbox.height, false); _mask = aliceVision::image::Image(coarseBbox.width, coarseBbox.height, true, 0); @@ -18,131 +20,99 @@ bool CoordinatesMap::build(const std::pair & panoramaSize, const geome size_t min_x = panoramaSize.first; size_t min_y = panoramaSize.second; - for (size_t y = 0; y < coarseBbox.height; y++) { + for(size_t y = 0; y < coarseBbox.height; y++) + { - size_t cy = y + coarseBbox.top; + size_t cy = y + coarseBbox.top; - for (size_t x = 0; x < coarseBbox.width; x++) { + for(size_t x = 0; x < coarseBbox.width; x++) + { - size_t cx = x + coarseBbox.left; + size_t cx = x + coarseBbox.left; - Vec3 ray = SphericalMapping::fromEquirectangular(Vec2(cx, cy), panoramaSize.first, panoramaSize.second); + Vec3 ray = SphericalMapping::fromEquirectangular(Vec2(cx, cy), panoramaSize.first, panoramaSize.second); - /** - * Check that this ray should be visible. - * This test is camera type dependent - */ - Vec3 transformedRay = pose(ray); - if (!intrinsics.isVisibleRay(transformedRay)) { - continue; - } + /** + * Check that this ray should be visible. + * This test is camera type dependent + */ + Vec3 transformedRay = pose(ray); + if(!intrinsics.isVisibleRay(transformedRay)) + { + continue; + } - /** - * Project this ray to camera pixel coordinates - */ - const Vec2 pix_disto = intrinsics.project(pose, ray, true); + /** + * Project this ray to camera pixel coordinates + */ + const Vec2 pix_disto = intrinsics.project(pose, ray, true); - /** - * Ignore invalid coordinates - */ - if (!intrinsics.isVisible(pix_disto)) { - continue; - } + /** + * Ignore invalid coordinates + */ + if(!intrinsics.isVisible(pix_disto)) + { + continue; + } - _coordinates(y, x) = pix_disto; - _mask(y, x) = 1; + _coordinates(y, x) = pix_disto; + _mask(y, x) = 1; - min_x = std::min(cx, min_x); - min_y = std::min(cy, min_y); - max_x = std::max(cx, max_x); - max_y = std::max(cy, max_y); - } + min_x = std::min(cx, min_x); + min_y = std::min(cy, min_y); + max_x = std::max(cx, max_x); + max_y = std::max(cy, max_y); + } } - + _offset_x = coarseBbox.left; _offset_y = coarseBbox.top; _boundingBox.left = min_x; _boundingBox.top = min_y; - _boundingBox.width = max_x - min_x + 1; + _boundingBox.width = max_x - min_x + 1; _boundingBox.height = max_y - min_y + 1; return true; } +bool CoordinatesMap::computeScale(double& result, float ratioUpscale) +{ -bool CoordinatesMap::containsPixels(const std::pair & panoramaSize, const geometry::Pose3 & pose, const aliceVision::camera::IntrinsicBase & intrinsics, const BoundingBox &coarseBbox) { - - for (size_t y = 0; y < coarseBbox.height; y++) { + std::vector scales; + size_t real_height = _coordinates.Height(); + size_t real_width = _coordinates.Width(); - size_t cy = y + coarseBbox.top; - - - for (size_t x = 0; x < coarseBbox.width; x++) { - - size_t cx = x + coarseBbox.left; - - Vec3 ray = SphericalMapping::fromEquirectangular(Vec2(cx, cy), panoramaSize.first, panoramaSize.second); - - /** - * Check that this ray should be visible. - * This test is camera type dependent - */ - Vec3 transformedRay = pose(ray); - if (!intrinsics.isVisibleRay(transformedRay)) { + for(int i = 1; i < real_height - 2; i++) + { + for(int j = 1; j < real_width - 2; j++) + { + if(!_mask(i, j) || !_mask(i, j + 1) || !_mask(i + 1, j)) + { continue; } - /** - * Project this ray to camera pixel coordinates - */ - const Vec2 pix_disto = intrinsics.project(pose, ray, true); + double dxx = _coordinates(i, j + 1).x() - _coordinates(i, j).x(); + double dxy = _coordinates(i + 1, j).x() - _coordinates(i, j).x(); + double dyx = _coordinates(i, j + 1).y() - _coordinates(i, j).y(); + double dyy = _coordinates(i + 1, j).y() - _coordinates(i, j).y(); - /** - * Ignore invalid coordinates - */ - if (!intrinsics.isVisible(pix_disto)) { + double det = std::abs(dxx * dyy - dxy * dyx); + if(det < 1e-12) continue; - } - return true; + scales.push_back(det); } } - - return false; -} -bool CoordinatesMap::computeScale(double & result, float ratioUpscale) { - - std::vector scales; - size_t real_height = _coordinates.Height(); - size_t real_width = _coordinates.Width(); - - for (int i = 1; i < real_height - 2; i++) { - for (int j = 1; j < real_width - 2; j++) { - if (!_mask(i, j) || !_mask(i, j + 1) || !_mask(i + 1, j)) { - continue; - } - - double dxx = _coordinates(i, j + 1).x() - _coordinates(i, j).x(); - double dxy = _coordinates(i + 1, j).x() - _coordinates(i, j).x(); - double dyx = _coordinates(i, j + 1).y() - _coordinates(i, j).y(); - double dyy = _coordinates(i + 1, j).y() - _coordinates(i, j).y(); - - double det = std::abs(dxx*dyy - dxy*dyx); - if (det < 1e-12) continue; - - scales.push_back(det); - } - } + if(scales.empty()) + return false; - if (scales.empty()) return false; + std::sort(scales.begin(), scales.end()); + int selected_index = int(floor(float(scales.size() - 1) * ratioUpscale)); + result = sqrt(scales[selected_index]); - std::sort(scales.begin(), scales.end()); - int selected_index = int(floor(float(scales.size() - 1) * ratioUpscale)); - result = sqrt(scales[selected_index]); - - return true; + return true; } -} \ No newline at end of file +} // namespace aliceVision \ No newline at end of file diff --git a/src/aliceVision/panorama/coordinatesMap.hpp b/src/aliceVision/panorama/coordinatesMap.hpp index f088d80167..69cdef54be 100644 --- a/src/aliceVision/panorama/coordinatesMap.hpp +++ b/src/aliceVision/panorama/coordinatesMap.hpp @@ -1,60 +1,44 @@ #pragma once - #include #include #include "boundingBox.hpp" -namespace aliceVision { +namespace aliceVision +{ -class CoordinatesMap { +class CoordinatesMap +{ public: - /** - * Build coordinates map given camera properties - * @param panoramaSize desired output panoramaSize - * @param pose the camera pose wrt an arbitrary reference frame - * @param intrinsics the camera intrinsics - */ - bool build(const std::pair & panoramaSize, const geometry::Pose3 & pose, const aliceVision::camera::IntrinsicBase & intrinsics, const BoundingBox &coarseBbox); - - /** - * Check if there is any valid pixel - * @param panoramaSize desired output panoramaSize - * @param pose the camera pose wrt an arbitrary reference frame - * @param intrinsics the camera intrinsics - */ - bool containsPixels(const std::pair & panoramaSize, const geometry::Pose3 & pose, const aliceVision::camera::IntrinsicBase & intrinsics, const BoundingBox &coarseBbox); - - bool computeScale(double & result, float ratioUpscale); - - size_t getOffsetX() const { - return _offset_x; - } - - size_t getOffsetY() const { - return _offset_y; - } - - BoundingBox getBoundingBox() const { - return _boundingBox; - } - - const aliceVision::image::Image & getCoordinates() const { - return _coordinates; - } - - const aliceVision::image::Image & getMask() const { - return _mask; - } + /** + * Build coordinates map given camera properties + * @param panoramaSize desired output panoramaSize + * @param pose the camera pose wrt an arbitrary reference frame + * @param intrinsics the camera intrinsics + */ + bool build(const std::pair& panoramaSize, const geometry::Pose3& pose, + const aliceVision::camera::IntrinsicBase& intrinsics, const BoundingBox& coarseBbox); + + bool computeScale(double& result, float ratioUpscale); + + size_t getOffsetX() const { return _offset_x; } + + size_t getOffsetY() const { return _offset_y; } + + BoundingBox getBoundingBox() const { return _boundingBox; } + + const aliceVision::image::Image& getCoordinates() const { return _coordinates; } + + const aliceVision::image::Image& getMask() const { return _mask; } private: - size_t _offset_x = 0; - size_t _offset_y = 0; + size_t _offset_x = 0; + size_t _offset_y = 0; - aliceVision::image::Image _coordinates; - aliceVision::image::Image _mask; - BoundingBox _boundingBox; + aliceVision::image::Image _coordinates; + aliceVision::image::Image _mask; + BoundingBox _boundingBox; }; -} \ No newline at end of file +} // namespace aliceVision \ No newline at end of file diff --git a/src/aliceVision/panorama/distance.cpp b/src/aliceVision/panorama/distance.cpp index 2a7d7504c8..02e7726143 100644 --- a/src/aliceVision/panorama/distance.cpp +++ b/src/aliceVision/panorama/distance.cpp @@ -2,36 +2,41 @@ #include "distance.hpp" -namespace aliceVision { +namespace aliceVision +{ -bool distanceToCenter(aliceVision::image::Image & _weights, const CoordinatesMap & map, int width, int height) { +bool distanceToCenter(aliceVision::image::Image& _weights, const CoordinatesMap& map, int width, int height) +{ - const aliceVision::image::Image & coordinates = map.getCoordinates(); - const aliceVision::image::Image & mask = map.getMask(); + const aliceVision::image::Image& coordinates = map.getCoordinates(); + const aliceVision::image::Image& mask = map.getMask(); float cx = width / 2.0f; float cy = height / 2.0f; _weights = aliceVision::image::Image(coordinates.Width(), coordinates.Height()); - for (int i = 0; i < _weights.Height(); i++) { - for (int j = 0; j < _weights.Width(); j++) { - + for(int i = 0; i < _weights.Height(); i++) + { + for(int j = 0; j < _weights.Width(); j++) + { + _weights(i, j) = 0.0f; bool valid = mask(i, j); - if (!valid) { + if(!valid) + { continue; } - const Vec2 & coords = coordinates(i, j); + const Vec2& coords = coordinates(i, j); float x = coords(0); float y = coords(1); float wx = 1.0f - std::abs((x - cx) / cx); float wy = 1.0f - std::abs((y - cy) / cy); - + _weights(i, j) = wx * wy; } } @@ -39,4 +44,95 @@ bool distanceToCenter(aliceVision::image::Image & _weights, const Coordin return true; } -} \ No newline at end of file +static inline int f(int x_i, int gi) noexcept +{ + return (x_i * x_i) + gi * gi; +} + +static inline int sep(int i, int u, int gi, int gu, int) noexcept +{ + return (u * u - i * i + gu * gu - gi * gi) / (2 * (u - i)); +} + +/// Code adapted from VFLib: https://github.com/vinniefalco/VFLib (Licence MIT) +bool computeDistanceMap(image::Image& distance, const image::Image& mask) +{ + + int width = mask.Width(); + int height = mask.Height(); + + int maxval = width + height; + image::Image buf(width, height); + + /* Per column distance 1D calculation */ + for(int j = 0; j < width; j++) + { + buf(0, j) = mask(0, j) ? 0 : maxval; + + /*Top to bottom accumulation */ + for(int i = 1; i < height; i++) + { + + buf(i, j) = mask(i, j) ? 0 : 1 + buf(i - 1, j); + } + + /*Bottom to top correction */ + for(int i = height - 2; i >= 0; i--) + { + + if(buf(i + 1, j) < buf(i, j)) + { + buf(i, j) = 1 + buf(i + 1, j); + } + } + } + + std::vector s(std::max(width, height)); + std::vector t(std::max(width, height)); + + /*Per row scan*/ + for(int i = 0; i < height; i++) + { + int q = 0; + s[0] = 0; + t[0] = 0; + + // scan 3 + for(int j = 1; j < width; j++) + { + while(q >= 0 && f(t[q] - s[q], buf(i, s[q])) > f(t[q] - j, buf(i, j))) + q--; + + if(q < 0) + { + q = 0; + s[0] = j; + } + else + { + int const w = 1 + sep(s[q], j, buf(i, s[q]), buf(i, j), maxval); + + if(w < width) + { + ++q; + s[q] = j; + t[q] = w; + } + } + } + + // scan 4 + for(int j = width - 1; j >= 0; --j) + { + int const d = f(j - s[q], buf(i, s[q])); + + distance(i, j) = d; + if(j == t[q]) + --q; + } + } + + return true; +} + +} // namespace aliceVision \ No newline at end of file diff --git a/src/aliceVision/panorama/distance.hpp b/src/aliceVision/panorama/distance.hpp index b27dd9dcd9..73f07ddff9 100644 --- a/src/aliceVision/panorama/distance.hpp +++ b/src/aliceVision/panorama/distance.hpp @@ -4,8 +4,10 @@ #include "coordinatesMap.hpp" -namespace aliceVision { +namespace aliceVision +{ -bool distanceToCenter(aliceVision::image::Image & _weights, const CoordinatesMap & map, int width, int height); +bool distanceToCenter(aliceVision::image::Image& _weights, const CoordinatesMap& map, int width, int height); +bool computeDistanceMap(image::Image& distance, const image::Image& mask); -} \ No newline at end of file +} // namespace aliceVision \ No newline at end of file diff --git a/src/aliceVision/panorama/feathering.cpp b/src/aliceVision/panorama/feathering.cpp new file mode 100644 index 0000000000..867a7c0e5a --- /dev/null +++ b/src/aliceVision/panorama/feathering.cpp @@ -0,0 +1,126 @@ +#include "feathering.hpp" + +namespace aliceVision +{ + +bool feathering(aliceVision::image::Image& output, + const aliceVision::image::Image& color, + const aliceVision::image::Image& inputMask) +{ + + std::vector> feathering; + std::vector> feathering_mask; + feathering.push_back(color); + feathering_mask.push_back(inputMask); + + int lvl = 0; + int width = color.Width(); + int height = color.Height(); + + while(1) + { + const image::Image& src = feathering[lvl]; + const image::Image& src_mask = feathering_mask[lvl]; + + image::Image half(width / 2, height / 2); + image::Image half_mask(width / 2, height / 2); + + for(int i = 0; i < half.Height(); i++) + { + + int di = i * 2; + for(int j = 0; j < half.Width(); j++) + { + int dj = j * 2; + + int count = 0; + half(i, j) = image::RGBfColor(0.0, 0.0, 0.0); + + if(src_mask(di, dj)) + { + half(i, j) += src(di, dj); + count++; + } + + if(src_mask(di, dj + 1)) + { + half(i, j) += src(di, dj + 1); + count++; + } + + if(src_mask(di + 1, dj)) + { + half(i, j) += src(di + 1, dj); + count++; + } + + if(src_mask(di + 1, dj + 1)) + { + half(i, j) += src(di + 1, dj + 1); + count++; + } + + if(count > 0) + { + half(i, j) /= float(count); + half_mask(i, j) = 1; + } + else + { + half_mask(i, j) = 0; + } + } + } + + feathering.push_back(half); + feathering_mask.push_back(half_mask); + + width = half.Width(); + height = half.Height(); + + if(width < 2 || height < 2) + break; + + lvl++; + } + + for(int lvl = feathering.size() - 2; lvl >= 0; lvl--) + { + + image::Image& src = feathering[lvl]; + image::Image& src_mask = feathering_mask[lvl]; + image::Image& ref = feathering[lvl + 1]; + image::Image& ref_mask = feathering_mask[lvl + 1]; + + for(int i = 0; i < src_mask.Height(); i++) + { + for(int j = 0; j < src_mask.Width(); j++) + { + if(!src_mask(i, j)) + { + int mi = i / 2; + int mj = j / 2; + + if(mi >= ref_mask.Height()) + { + mi = ref_mask.Height() - 1; + } + + if(mj >= ref_mask.Width()) + { + mj = ref_mask.Width() - 1; + } + + src_mask(i, j) = ref_mask(mi, mj); + src(i, j) = ref(mi, mj); + } + } + } + } + + output = feathering[0]; + + return true; +} + +} // namespace aliceVision \ No newline at end of file diff --git a/src/aliceVision/panorama/feathering.hpp b/src/aliceVision/panorama/feathering.hpp new file mode 100644 index 0000000000..8146b91177 --- /dev/null +++ b/src/aliceVision/panorama/feathering.hpp @@ -0,0 +1,12 @@ +#pragma once + +#include + +namespace aliceVision +{ + +bool feathering(aliceVision::image::Image& output, + const aliceVision::image::Image& color, + const aliceVision::image::Image& inputMask); + +} \ No newline at end of file diff --git a/src/aliceVision/panorama/gaussian.cpp b/src/aliceVision/panorama/gaussian.cpp index e009a1ec95..126b409456 100644 --- a/src/aliceVision/panorama/gaussian.cpp +++ b/src/aliceVision/panorama/gaussian.cpp @@ -2,11 +2,13 @@ #include -namespace aliceVision { +namespace aliceVision +{ -GaussianPyramidNoMask::GaussianPyramidNoMask(const size_t width_base, const size_t height_base, const size_t limit_scales) : - _width_base(width_base), - _height_base(height_base) +GaussianPyramidNoMask::GaussianPyramidNoMask(const size_t width_base, const size_t height_base, + const size_t limit_scales) + : _width_base(width_base) + , _height_base(height_base) { /** * Compute optimal scale @@ -15,27 +17,29 @@ GaussianPyramidNoMask::GaussianPyramidNoMask(const size_t width_base, const size size_t min_dim = std::min(_width_base, _height_base); size_t min_size = 32; _scales = std::min(limit_scales, static_cast(floor(log2(double(min_dim) / float(min_size))))); - /** * Create pyramid **/ size_t new_width = _width_base; size_t new_height = _height_base; - for (int i = 0; i < _scales; i++) { + for(int i = 0; i < _scales; i++) + { - _pyramid_color.push_back(image::Image(new_width, new_height, true, image::RGBfColor(0))); - _filter_buffer.push_back(image::Image(new_width, new_height, true, image::RGBfColor(0))); - new_height /= 2; - new_width /= 2; + _pyramid_color.push_back(image::Image(new_width, new_height, true, image::RGBfColor(0))); + _filter_buffer.push_back(image::Image(new_width, new_height, true, image::RGBfColor(0))); + new_height /= 2; + new_width /= 2; } } -bool GaussianPyramidNoMask::process(const image::Image & input) { - - if (input.Height() != _pyramid_color[0].Height()) return false; - if (input.Width() != _pyramid_color[0].Width()) return false; +bool GaussianPyramidNoMask::process(const image::Image& input) +{ + if(input.Height() != _pyramid_color[0].Height()) + return false; + if(input.Width() != _pyramid_color[0].Width()) + return false; /** * Kernel @@ -43,34 +47,38 @@ bool GaussianPyramidNoMask::process(const image::Image & input oiio::ImageBuf K; oiio::ImageBufAlgo::make_kernel(K, "gaussian", 5, 5); - /** + /** * Build pyramid - */ + */ _pyramid_color[0] = input; - for (int lvl = 0; lvl < _scales - 1; lvl++) { - - const image::Image & source = _pyramid_color[lvl]; - image::Image & dst = _filter_buffer[lvl]; + for(int lvl = 0; lvl < _scales - 1; lvl++) + { + + const image::Image& source = _pyramid_color[lvl]; + image::Image& dst = _filter_buffer[lvl]; oiio::ImageSpec spec(source.Width(), source.Height(), 3, oiio::TypeDesc::FLOAT); const oiio::ImageBuf inBuf(spec, const_cast(source.data())); - oiio::ImageBuf outBuf(spec, dst.data()); + oiio::ImageBuf outBuf(spec, dst.data()); oiio::ImageBufAlgo::convolve(outBuf, inBuf, K); - downscale(_pyramid_color[lvl + 1], _filter_buffer[lvl]); + downscale(_pyramid_color[lvl + 1], _filter_buffer[lvl]); } return true; } +bool GaussianPyramidNoMask::downscale(image::Image& output, + const image::Image& input) +{ -bool GaussianPyramidNoMask::downscale(image::Image & output, const image::Image & input) { - - for (int i = 0; i < output.Height(); i++) { + for(int i = 0; i < output.Height(); i++) + { int ui = i * 2; - for (int j = 0; j < output.Width(); j++) { + for(int j = 0; j < output.Width(); j++) + { int uj = j * 2; output(i, j) = input(ui, uj); @@ -80,4 +88,4 @@ bool GaussianPyramidNoMask::downscale(image::Image & output, c return true; } -} \ No newline at end of file +} // namespace aliceVision \ No newline at end of file diff --git a/src/aliceVision/panorama/gaussian.hpp b/src/aliceVision/panorama/gaussian.hpp index 7f242b5547..42546507bc 100644 --- a/src/aliceVision/panorama/gaussian.hpp +++ b/src/aliceVision/panorama/gaussian.hpp @@ -2,35 +2,170 @@ #include -namespace aliceVision { +namespace aliceVision +{ -class GaussianPyramidNoMask { +class GaussianPyramidNoMask +{ public: - GaussianPyramidNoMask(const size_t width_base, const size_t height_base, const size_t limit_scales = 64); + GaussianPyramidNoMask(const size_t width_base, const size_t height_base, const size_t limit_scales = 64); - bool process(const image::Image & input); + bool process(const image::Image& input); + bool downscale(image::Image& output, const image::Image& input); - bool downscale(image::Image & output, const image::Image & input); + const size_t getScalesCount() const { return _scales; } - const size_t getScalesCount() const { - return _scales; - } + const std::vector>& getPyramidColor() const { return _pyramid_color; } - const std::vector> & getPyramidColor() const { - return _pyramid_color; - } - - std::vector> & getPyramidColor() { - return _pyramid_color; - } + std::vector>& getPyramidColor() { return _pyramid_color; } protected: - std::vector> _pyramid_color; - std::vector> _filter_buffer; - size_t _width_base; - size_t _height_base; - size_t _scales; + std::vector> _pyramid_color; + std::vector> _filter_buffer; + size_t _width_base; + size_t _height_base; + size_t _scales; }; -} \ No newline at end of file +template +inline void convolveRow(typename image::Image::RowXpr output_row, typename image::Image::ConstRowXpr input_row, + const Eigen::Matrix& kernel, bool loop) +{ + + const int radius = 2; + + for(int j = 0; j < input_row.cols(); j++) + { + + T sum = T(); + float sumw = 0.0f; + + for(int k = 0; k < kernel.size(); k++) + { + + float w = kernel(k); + int col = j + k - radius; + + /* mirror 5432 | 123456 | 5432 */ + + if(!loop) + { + if(col < 0) + { + col = -col; + } + + if(col >= input_row.cols()) + { + col = input_row.cols() - 1 - (col + 1 - input_row.cols()); + } + } + else + { + if(col < 0) + { + col = input_row.cols() + col; + } + + if(col >= input_row.cols()) + { + col = col - input_row.cols(); + } + } + + sum += w * input_row(col); + sumw += w; + } + + output_row(j) = sum / sumw; + } +} + +template +inline void convolveColumns(typename image::Image::RowXpr output_row, const image::Image& input_rows, + const Eigen::Matrix& kernel) +{ + + for(int j = 0; j < output_row.cols(); j++) + { + + T sum = T(); + float sumw = 0.0f; + + for(int k = 0; k < kernel.size(); k++) + { + + float w = kernel(k); + sum += w * input_rows(k, j); + sumw += w; + } + + output_row(j) = sum / sumw; + } +} + +template +bool convolveGaussian5x5(image::Image& output, const image::Image& input, bool loop = false) +{ + + if(output.size() != input.size()) + { + return false; + } + + Eigen::Matrix kernel; + kernel[0] = 1.0f; + kernel[1] = 4.0f; + kernel[2] = 6.0f; + kernel[3] = 4.0f; + kernel[4] = 1.0f; + kernel = kernel / kernel.sum(); + + image::Image buf(output.Width(), 5); + + int radius = 2; + + convolveRow(buf.row(0), input.row(2), kernel, loop); + convolveRow(buf.row(1), input.row(1), kernel, loop); + convolveRow(buf.row(2), input.row(0), kernel, loop); + convolveRow(buf.row(3), input.row(1), kernel, loop); + convolveRow(buf.row(4), input.row(2), kernel, loop); + + for(int i = 0; i < output.Height() - 3; i++) + { + + convolveColumns(output.row(i), buf, kernel); + + buf.row(0) = buf.row(1); + buf.row(1) = buf.row(2); + buf.row(2) = buf.row(3); + buf.row(3) = buf.row(4); + convolveRow(buf.row(4), input.row(i + 3), kernel, loop); + } + + /** + current row : -5 -4 -3 -2 -1 + next 1 : -4 -3 -2 -1 -2 + next 2 : -3 -2 -1 -2 -3 + */ + convolveColumns(output.row(output.Height() - 3), buf, kernel); + + buf.row(0) = buf.row(1); + buf.row(1) = buf.row(2); + buf.row(2) = buf.row(3); + buf.row(3) = buf.row(4); + convolveRow(buf.row(4), input.row(output.Height() - 2), kernel, loop); + convolveColumns(output.row(output.Height() - 2), buf, kernel); + + buf.row(0) = buf.row(1); + buf.row(1) = buf.row(2); + buf.row(2) = buf.row(3); + buf.row(3) = buf.row(4); + convolveRow(buf.row(4), input.row(output.Height() - 3), kernel, loop); + convolveColumns(output.row(output.Height() - 1), buf, kernel); + + return true; +} + +} // namespace aliceVision \ No newline at end of file diff --git a/src/aliceVision/panorama/graphcut.hpp b/src/aliceVision/panorama/graphcut.hpp new file mode 100644 index 0000000000..3fdcdeb073 --- /dev/null +++ b/src/aliceVision/panorama/graphcut.hpp @@ -0,0 +1,872 @@ +#pragma once + +#include +#include + +#include + +#include "distance.hpp" + +namespace aliceVision +{ + +/** + * @brief Maxflow computation based on a standard Adjacency List graph reprensentation. + * + * @see MaxFlow_CSR which use less memory. + */ +class MaxFlow_AdjList +{ +public: + using NodeType = int; + using ValueType = float; + + using Traits = boost::adjacency_list_traits; + using edge_descriptor = typename Traits::edge_descriptor; + using vertex_descriptor = typename Traits::vertex_descriptor; + using vertex_size_type = typename Traits::vertices_size_type; + struct Edge + { + ValueType capacity{}; + ValueType residual{}; + edge_descriptor reverse; + }; + using Graph = boost::adjacency_list; + using VertexIterator = typename boost::graph_traits::vertex_iterator; + +public: + explicit MaxFlow_AdjList(size_t numNodes) + : _graph(numNodes + 2) + , _S(NodeType(numNodes)) + , _T(NodeType(numNodes + 1)) + { + VertexIterator vi, vi_end; + for(boost::tie(vi, vi_end) = vertices(_graph); vi != vi_end; ++vi) + { + _graph.m_vertices[*vi].m_out_edges.reserve(9); + } + _graph.m_vertices[numNodes].m_out_edges.reserve(numNodes); + _graph.m_vertices[numNodes + 1].m_out_edges.reserve(numNodes); + } + + inline void addNodeToSource(NodeType n, ValueType source) + { + assert(source >= 0); + + edge_descriptor edge(boost::add_edge(_S, n, _graph).first); + edge_descriptor reverseEdge(boost::add_edge(n, _S, _graph).first); + + _graph[edge].capacity = source; + _graph[edge].reverse = reverseEdge; + _graph[reverseEdge].reverse = edge; + _graph[reverseEdge].capacity = source; + } + + inline void addNodeToSink(NodeType n, ValueType sink) + { + assert(sink >= 0); + + edge_descriptor edge(boost::add_edge(_T, n, _graph).first); + edge_descriptor reverseEdge(boost::add_edge(n, _T, _graph).first); + + _graph[edge].capacity = sink; + _graph[edge].reverse = reverseEdge; + _graph[reverseEdge].reverse = edge; + _graph[reverseEdge].capacity = sink; + } + + inline void addEdge(NodeType n1, NodeType n2, ValueType capacity, ValueType reverseCapacity) + { + assert(capacity >= 0 && reverseCapacity >= 0); + + edge_descriptor edge(boost::add_edge(n1, n2, _graph).first); + edge_descriptor reverseEdge(boost::add_edge(n2, n1, _graph).first); + _graph[edge].capacity = capacity; + _graph[edge].reverse = reverseEdge; + + _graph[reverseEdge].capacity = reverseCapacity; + _graph[reverseEdge].reverse = edge; + } + + void printStats() const; + void printColorStats() const; + + inline ValueType compute() + { + vertex_size_type nbVertices(boost::num_vertices(_graph)); + _color.resize(nbVertices, boost::white_color); + std::vector pred(nbVertices); + std::vector dist(nbVertices); + + ValueType v = boost::boykov_kolmogorov_max_flow(_graph, boost::get(&Edge::capacity, _graph), + boost::get(&Edge::residual, _graph), + boost::get(&Edge::reverse, _graph), &pred[0], &_color[0], + &dist[0], boost::get(boost::vertex_index, _graph), _S, _T); + + return v; + } + + /// is empty + inline bool isSource(NodeType n) const { return (_color[n] == boost::black_color); } + /// is full + inline bool isTarget(NodeType n) const { return (_color[n] == boost::white_color); } + +protected: + Graph _graph; + std::vector _color; + const NodeType _S; //< emptyness + const NodeType _T; //< fullness +}; + +bool computeSeamsMap(image::Image& seams, const image::Image& labels) +{ + + if(seams.size() != labels.size()) + { + return false; + } + + seams.fill(0); + + for(int j = 1; j < labels.Width() - 1; j++) + { + IndexT label = labels(0, j); + IndexT same = true; + + same &= (labels(0, j - 1) == label); + same &= (labels(0, j + 1) == label); + same &= (labels(1, j - 1) == label); + same &= (labels(1, j) == label); + same &= (labels(1, j + 1) == label); + + if(same) + { + continue; + } + + seams(0, j) = 255; + } + + int lastrow = labels.Height() - 1; + for(int j = 1; j < labels.Width() - 1; j++) + { + IndexT label = labels(lastrow, j); + IndexT same = true; + + same &= (labels(lastrow - 1, j - 1) == label); + same &= (labels(lastrow - 1, j + 1) == label); + same &= (labels(lastrow, j - 1) == label); + same &= (labels(lastrow, j) == label); + same &= (labels(lastrow, j + 1) == label); + + if(same) + { + continue; + } + + seams(lastrow, j) = 255; + } + + for(int i = 1; i < labels.Height() - 1; i++) + { + + for(int j = 1; j < labels.Width() - 1; j++) + { + + IndexT label = labels(i, j); + IndexT same = true; + + same &= (labels(i - 1, j - 1) == label); + same &= (labels(i - 1, j) == label); + same &= (labels(i - 1, j + 1) == label); + same &= (labels(i, j - 1) == label); + same &= (labels(i, j + 1) == label); + same &= (labels(i + 1, j - 1) == label); + same &= (labels(i + 1, j) == label); + same &= (labels(i + 1, j + 1) == label); + + if(same) + { + continue; + } + + seams(i, j) = 255; + } + } + + return true; +} + +class GraphcutSeams +{ +public: + struct Rect + { + int l; + int t; + int w; + int h; + }; + + using PixelInfo = std::pair; + using ImageOwners = image::Image>; + +public: + GraphcutSeams(size_t outputWidth, size_t outputHeight) + : _owners(outputWidth, outputHeight, true) + , _labels(outputWidth, outputHeight, true, 0) + , _original_labels(outputWidth, outputHeight, true, 0) + , _distancesSeams(outputWidth, outputHeight, true, 0) + , _maximal_distance_change(outputWidth + outputHeight) + { + } + + virtual ~GraphcutSeams() = default; + + void setOriginalLabels(const image::Image& existing_labels) + { + + _labels = existing_labels; + _original_labels = existing_labels; + + image::Image seams(_labels.Width(), _labels.Height()); + computeSeamsMap(seams, _labels); + computeDistanceMap(_distancesSeams, seams); + } + + virtual bool append(const aliceVision::image::Image& input, + const aliceVision::image::Image& inputMask, IndexT currentIndex, size_t offset_x, + size_t offset_y) + { + + if(inputMask.size() != input.size()) + { + return false; + } + + Rect rect; + + rect.l = offset_x; + rect.t = offset_y; + rect.w = input.Width() + 1; + rect.h = input.Height() + 1; + + /*Extend rect for borders*/ + rect.l = std::max(0, rect.l - 3); + rect.t = std::max(0, rect.t - 3); + rect.w = rect.w + 6; + rect.h = rect.h + 6; + if(rect.t + rect.h > _owners.Height()) + { + rect.h = _owners.Height() - rect.t; + } + + _rects[currentIndex] = rect; + + /* + _owners will get for each pixel of the panorama a list of pixels + in the sources which may have seen this point. + */ + for(int i = 0; i < input.Height(); i++) + { + + int di = i + offset_y; + + for(int j = 0; j < input.Width(); j++) + { + + if(!inputMask(i, j)) + continue; + + int dj = j + offset_x; + if(dj >= _owners.Width()) + { + dj = dj - _owners.Width(); + } + + PixelInfo info; + info.first = currentIndex; + info.second = input(i, j); + + /* If too far away from seam, do not add a contender */ + int dist = _distancesSeams(di, dj); + if(dist > _maximal_distance_change + 10) + { + continue; + } + + _owners(di, dj).push_back(info); + } + } + + return true; + } + + void setMaximalDistance(int dist) { _maximal_distance_change = dist; } + + bool process() + { + + for(int i = 0; i < 10; i++) + { + + /*For each possible label, try to extends its domination on the label's world */ + bool change = false; + + for(auto& info : _rects) + { + + ALICEVISION_LOG_INFO("Graphcut expansion (iteration " << i << ") for label " << info.first); + + int p1 = info.second.l; + int w1 = info.second.w; + int p2 = 0; + int w2 = 0; + + if(p1 + w1 > _labels.Width()) + { + w1 = _labels.Width() - p1; + p2 = 0; + w2 = info.second.w - w1; + } + + Eigen::Matrix backup_1 = + _labels.block(info.second.t, p1, info.second.h, w1); + Eigen::Matrix backup_2 = + _labels.block(info.second.t, p2, info.second.h, w2); + + double base_cost = cost(info.first); + alphaExpansion(info.first); + double new_cost = cost(info.first); + + if(new_cost > base_cost) + { + _labels.block(info.second.t, p1, info.second.h, w1) = backup_1; + _labels.block(info.second.t, p2, info.second.h, w2) = backup_2; + } + else if(new_cost < base_cost) + { + change = true; + } + } + + if(!change) + { + break; + } + } + + return true; + } + + double cost(IndexT currentLabel) + { + + Rect rect = _rects[currentLabel]; + + double cost = 0.0; + + for(int i = 0; i < rect.h - 1; i++) + { + + int y = rect.t + i; + int yp = y + 1; + + for(int j = 0; j < rect.w; j++) + { + + int x = rect.l + j; + if(x >= _owners.Width()) + { + x = x - _owners.Width(); + } + + int xp = x + 1; + if(xp >= _owners.Width()) + { + xp = xp - _owners.Width(); + } + + IndexT label = _labels(y, x); + IndexT labelx = _labels(y, xp); + IndexT labely = _labels(yp, x); + + if(label == UndefinedIndexT) + continue; + if(labelx == UndefinedIndexT) + continue; + if(labely == UndefinedIndexT) + continue; + + if(label == labelx) + { + continue; + } + + image::RGBfColor CColorLC; + image::RGBfColor CColorLX; + image::RGBfColor CColorLY; + bool hasCLC = false; + bool hasCLX = false; + bool hasCLY = false; + + for(int l = 0; l < _owners(y, x).size(); l++) + { + if(_owners(y, x)[l].first == label) + { + hasCLC = true; + CColorLC = _owners(y, x)[l].second; + } + + if(_owners(y, x)[l].first == labelx) + { + hasCLX = true; + CColorLX = _owners(y, x)[l].second; + } + + if(_owners(y, x)[l].first == labely) + { + hasCLY = true; + CColorLY = _owners(y, x)[l].second; + } + } + + image::RGBfColor XColorLC; + image::RGBfColor XColorLX; + bool hasXLC = false; + bool hasXLX = false; + + for(int l = 0; l < _owners(y, xp).size(); l++) + { + if(_owners(y, xp)[l].first == label) + { + hasXLC = true; + XColorLC = _owners(y, xp)[l].second; + } + + if(_owners(y, xp)[l].first == labelx) + { + hasXLX = true; + XColorLX = _owners(y, xp)[l].second; + } + } + + image::RGBfColor YColorLC; + image::RGBfColor YColorLY; + bool hasYLC = false; + bool hasYLY = false; + + for(int l = 0; l < _owners(yp, x).size(); l++) + { + if(_owners(yp, x)[l].first == label) + { + hasYLC = true; + YColorLC = _owners(yp, x)[l].second; + } + + if(_owners(yp, x)[l].first == labely) + { + hasYLY = true; + YColorLY = _owners(yp, x)[l].second; + } + } + + if(!hasCLC || !hasXLX || !hasYLY) + { + continue; + } + + if(!hasCLX) + { + CColorLX = CColorLC; + } + + if(!hasCLY) + { + CColorLY = CColorLC; + } + + if(!hasXLC) + { + XColorLC = XColorLX; + } + + if(!hasYLC) + { + YColorLC = YColorLY; + } + + cost += (CColorLC - CColorLX).norm(); + cost += (CColorLC - CColorLY).norm(); + cost += (XColorLC - XColorLX).norm(); + cost += (YColorLC - YColorLY).norm(); + } + } + + return cost; + } + + bool alphaExpansion(IndexT currentLabel) + { + + Rect rect = _rects[currentLabel]; + + image::Image mask(rect.w, rect.h, true, 0); + image::Image ids(rect.w, rect.h, true, -1); + image::Image color_label(rect.w, rect.h, true, image::RGBfColor(0.0f, 0.0f, 0.0f)); + image::Image color_other(rect.w, rect.h, true, image::RGBfColor(0.0f, 0.0f, 0.0f)); + + /*Compute distance map to seams*/ + image::Image distanceMap(rect.w, rect.h); + { + image::Image binarizedWorld(rect.w, rect.h); + + for(int i = 0; i < rect.h; i++) + { + int y = rect.t + i; + + for(int j = 0; j < rect.w; j++) + { + + int x = rect.l + j; + if(x >= _owners.Width()) + { + x = x - _owners.Width(); + } + + IndexT label = _original_labels(y, x); + if(label == currentLabel) + { + binarizedWorld(i, j) = 1; + } + else + { + binarizedWorld(i, j) = 0; + } + } + } + + image::Image seams(rect.w, rect.h); + if(!computeSeamsMap(seams, binarizedWorld)) + { + return false; + } + + if(!computeDistanceMap(distanceMap, seams)) + { + return false; + } + } + + /* + A warped input has valid pixels only in some parts of the final image. + Rect is the bounding box of these valid pixels. + Let's build a mask : + - 0 if the pixel is not viewed by anyone + - 1 if the pixel is viewed by the current label alpha + - 2 if the pixel is viewed by *another* label and this label is marked as current valid label + - 3 if the pixel is 1 + 2 : the pixel is not selected as alpha territory, but alpha is looking at it + */ + for(int i = 0; i < rect.h; i++) + { + + int y = rect.t + i; + + for(int j = 0; j < rect.w; j++) + { + + int x = rect.l + j; + if(x >= _owners.Width()) + { + x = x - _owners.Width(); + } + + std::vector& infos = _owners(y, x); + IndexT label = _labels(y, x); + + image::RGBfColor currentColor; + image::RGBfColor otherColor; + + int dist = distanceMap(i, j); + + /* Loop over observations */ + for(int l = 0; l < infos.size(); l++) + { + + if(dist > _maximal_distance_change) + { + + if(infos[l].first == label) + { + if(label == currentLabel) + { + mask(i, j) = 1; + currentColor = infos[l].second; + } + else + { + mask(i, j) = 2; + otherColor = infos[l].second; + } + } + } + else + { + if(infos[l].first == currentLabel) + { + mask(i, j) |= 1; + currentColor = infos[l].second; + } + else if(infos[l].first == label) + { + mask(i, j) |= 2; + otherColor = infos[l].second; + } + } + } + + /* + If the pixel may be a new kingdom for alpha ! + */ + if(mask(i, j) == 1) + { + color_label(i, j) = currentColor; + color_other(i, j) = currentColor; + } + else if(mask(i, j) == 2) + { + color_label(i, j) = otherColor; + color_other(i, j) = otherColor; + } + else if(mask(i, j) == 3) + { + color_label(i, j) = currentColor; + color_other(i, j) = otherColor; + } + } + } + + /* + The rectangle is a grid. + However we want to ignore a lot of pixel. + Let's create an index per valid pixels for graph cut reference + */ + int count = 0; + for(int i = 0; i < rect.h; i++) + { + for(int j = 0; j < rect.w; j++) + { + if(mask(i, j) == 0) + { + continue; + } + + ids(i, j) = count; + count++; + } + } + + /*Create graph*/ + MaxFlow_AdjList gc(count); + size_t countValid = 0; + + for(int i = 0; i < rect.h; i++) + { + for(int j = 0; j < rect.w; j++) + { + + /* If this pixel is not valid, ignore */ + if(mask(i, j) == 0) + { + continue; + } + + /* Get this pixel ID */ + int node_id = ids(i, j); + + int im1 = std::max(i - 1, 0); + int jm1 = std::max(j - 1, 0); + int ip1 = std::min(i + 1, rect.h - 1); + int jp1 = std::min(j + 1, rect.w - 1); + + if(mask(i, j) == 1) + { + + /* Only add nodes close to borders */ + if(mask(im1, jm1) == 1 && mask(im1, j) == 1 && mask(im1, jp1) == 1 && mask(i, jm1) == 1 && + mask(i, jp1) == 1 && mask(ip1, jm1) == 1 && mask(ip1, j) == 1 && mask(ip1, jp1) == 1) + { + continue; + } + + /* + This pixel is only seen by alpha. + Enforce its domination by stating that removing this pixel + from alpha territoy is infinitly costly (impossible). + */ + gc.addNodeToSource(node_id, 100000); + } + else if(mask(i, j) == 2) + { + /* Only add nodes close to borders */ + if(mask(im1, jm1) == 2 && mask(im1, j) == 2 && mask(im1, jp1) == 2 && mask(i, jm1) == 2 && + mask(i, jp1) == 2 && mask(ip1, jm1) == 2 && mask(ip1, j) == 2 && mask(ip1, jp1) == 2) + { + continue; + } + + /* + This pixel is only seen by an ennemy. + Enforce its domination by stating that removing this pixel + from ennemy territory is infinitly costly (impossible). + */ + gc.addNodeToSink(node_id, 100000); + } + else if(mask(i, j) == 3) + { + + /* + This pixel is seen by both alpha and enemies but is owned by ennemy. + Make sure that changing node owner will have no direct cost. + Connect it to both alpha and ennemy for the moment + (Graph cut will not allow a pixel to have both owners at the end). + */ + gc.addNodeToSource(node_id, 0); + gc.addNodeToSink(node_id, 0); + countValid++; + } + } + } + + if(countValid == 0) + { + /* We have no possibility for territory expansion */ + /* let's exit */ + return true; + } + + /* + Loop over alpha bounding box. + Let's define the transition cost. + When two neighboor pixels have different labels, there is a seam (border) cost. + Graph cut will try to make sure the territory will have a minimal border cost + */ + for(int i = 0; i < rect.h; i++) + { + for(int j = 0; j < rect.w; j++) + { + + if(mask(i, j) == 0) + { + continue; + } + + int node_id = ids(i, j); + + /* Make sure it is possible to estimate this horizontal border */ + if(i < mask.Height() - 1) + { + + /* Make sure the other pixel is owned by someone */ + if(mask(i + 1, j)) + { + + int other_node_id = ids(i + 1, j); + float w = 1000; + + if(((mask(i, j) & 1) && (mask(i + 1, j) & 2)) || ((mask(i, j) & 2) && (mask(i + 1, j) & 1))) + { + float d1 = (color_label(i, j) - color_other(i, j)).norm(); + float d2 = (color_label(i + 1, j) - color_other(i + 1, j)).norm(); + + d1 = std::min(2.0f, d1); + d2 = std::min(2.0f, d2); + + w = (d1 + d2) * 100.0 + 1.0; + } + + gc.addEdge(node_id, other_node_id, w, w); + } + } + + if(j < mask.Width() - 1) + { + + if(mask(i, j + 1)) + { + + int other_node_id = ids(i, j + 1); + float w = 1000; + + if(((mask(i, j) & 1) && (mask(i, j + 1) & 2)) || ((mask(i, j) & 2) && (mask(i, j + 1) & 1))) + { + float d1 = (color_label(i, j) - color_other(i, j)).norm(); + float d2 = (color_label(i, j + 1) - color_other(i, j + 1)).norm(); + w = (d1 + d2) * 100.0 + 1.0; + } + + gc.addEdge(node_id, other_node_id, w, w); + } + } + } + } + + gc.compute(); + + int changeCount = 0; + for(int i = 0; i < rect.h; i++) + { + + int y = rect.t + i; + + for(int j = 0; j < rect.w; j++) + { + + int x = rect.l + j; + if(x >= _owners.Width()) + { + x = x - _owners.Width(); + } + + IndexT label = _labels(y, x); + int id = ids(i, j); + + if(gc.isSource(id)) + { + + if(label != currentLabel) + { + changeCount++; + } + + _labels(y, x) = currentLabel; + } + } + } + + return true; + } + + const image::Image& getLabels() { return _labels; } + +private: + std::map _rects; + ImageOwners _owners; + image::Image _labels; + image::Image _original_labels; + image::Image _distancesSeams; + size_t _maximal_distance_change; +}; + +} // namespace aliceVision \ No newline at end of file diff --git a/src/aliceVision/panorama/imageOps.cpp b/src/aliceVision/panorama/imageOps.cpp new file mode 100644 index 0000000000..2667537afc --- /dev/null +++ b/src/aliceVision/panorama/imageOps.cpp @@ -0,0 +1,36 @@ +#include "imageOps.hpp" + +namespace aliceVision +{ + +void removeNegativeValues(aliceVision::image::Image& img) +{ + for(int i = 0; i < img.Height(); i++) + { + for(int j = 0; j < img.Width(); j++) + { + image::RGBfColor& pix = img(i, j); + image::RGBfColor rpix; + rpix.r() = std::exp(pix.r()); + rpix.g() = std::exp(pix.g()); + rpix.b() = std::exp(pix.b()); + + if(rpix.r() < 0.0) + { + pix.r() = 0.0; + } + + if(rpix.g() < 0.0) + { + pix.g() = 0.0; + } + + if(rpix.b() < 0.0) + { + pix.b() = 0.0; + } + } + } +} + +} // namespace aliceVision \ No newline at end of file diff --git a/src/aliceVision/panorama/imageOps.hpp b/src/aliceVision/panorama/imageOps.hpp new file mode 100644 index 0000000000..d19472742f --- /dev/null +++ b/src/aliceVision/panorama/imageOps.hpp @@ -0,0 +1,116 @@ +#pragma once + +#include + +namespace aliceVision +{ + +template +bool downscale(aliceVision::image::Image& outputColor, const aliceVision::image::Image& inputColor) +{ + + size_t output_width = inputColor.Width() / 2; + size_t output_height = inputColor.Height() / 2; + + for(int i = 0; i < output_height; i++) + { + for(int j = 0; j < output_width; j++) + { + outputColor(i, j) = inputColor(i * 2, j * 2); + } + } + + return true; +} + +template +bool upscale(aliceVision::image::Image& outputColor, const aliceVision::image::Image& inputColor) +{ + + size_t width = inputColor.Width(); + size_t height = inputColor.Height(); + + for(int i = 0; i < height; i++) + { + + int di = i * 2; + + for(int j = 0; j < width; j++) + { + int dj = j * 2; + + outputColor(di, dj) = T(); + outputColor(di, dj + 1) = T(); + outputColor(di + 1, dj) = T(); + outputColor(di + 1, dj + 1) = inputColor(i, j); + } + } + + return true; +} + +template +bool substract(aliceVision::image::Image& AminusB, const aliceVision::image::Image& A, + const aliceVision::image::Image& B) +{ + + size_t width = AminusB.Width(); + size_t height = AminusB.Height(); + + if(AminusB.size() != A.size()) + { + return false; + } + + if(AminusB.size() != B.size()) + { + return false; + } + + for(int i = 0; i < height; i++) + { + + for(int j = 0; j < width; j++) + { + + AminusB(i, j) = A(i, j) - B(i, j); + } + } + + return true; +} + +template +bool addition(aliceVision::image::Image& AplusB, const aliceVision::image::Image& A, + const aliceVision::image::Image& B) +{ + + size_t width = AplusB.Width(); + size_t height = AplusB.Height(); + + if(AplusB.size() != A.size()) + { + return false; + } + + if(AplusB.size() != B.size()) + { + return false; + } + + for(int i = 0; i < height; i++) + { + + for(int j = 0; j < width; j++) + { + + AplusB(i, j) = A(i, j) + B(i, j); + } + } + + return true; +} + +void removeNegativeValues(aliceVision::image::Image& img); + +} // namespace aliceVision \ No newline at end of file diff --git a/src/aliceVision/panorama/laplacianCompositer.hpp b/src/aliceVision/panorama/laplacianCompositer.hpp new file mode 100644 index 0000000000..5131b0dfc6 --- /dev/null +++ b/src/aliceVision/panorama/laplacianCompositer.hpp @@ -0,0 +1,155 @@ +#pragma once + +#include "compositer.hpp" + +namespace aliceVision +{ + +template +bool makeImagePyramidCompatible(image::Image& output, size_t& out_offset_x, size_t& out_offset_y, + const image::Image& input, size_t offset_x, size_t offset_y, size_t num_levels) +{ + + if(num_levels == 0) + { + return false; + } + + double max_scale = 1.0 / pow(2.0, num_levels - 1); + + double low_offset_x = double(offset_x) * max_scale; + double low_offset_y = double(offset_y) * max_scale; + + /*Make sure offset is integer even at the lowest level*/ + double corrected_low_offset_x = floor(low_offset_x); + double corrected_low_offset_y = floor(low_offset_y); + + /*Add some borders on the top and left to make sure mask can be smoothed*/ + corrected_low_offset_x = std::max(0.0, corrected_low_offset_x - 3.0); + corrected_low_offset_y = std::max(0.0, corrected_low_offset_y - 3.0); + + /*Compute offset at largest level*/ + out_offset_x = size_t(corrected_low_offset_x / max_scale); + out_offset_y = size_t(corrected_low_offset_y / max_scale); + + /*Compute difference*/ + double doffset_x = double(offset_x) - double(out_offset_x); + double doffset_y = double(offset_y) - double(out_offset_y); + + /* update size with border update */ + double large_width = double(input.Width()) + doffset_x; + double large_height = double(input.Height()) + doffset_y; + + /* compute size at largest scale */ + double low_width = large_width * max_scale; + double low_height = large_height * max_scale; + + /*Make sure width is integer event at the lowest level*/ + double corrected_low_width = ceil(low_width); + double corrected_low_height = ceil(low_height); + + /*Add some borders on the right and bottom to make sure mask can be smoothed*/ + corrected_low_width = corrected_low_width + 3; + corrected_low_height = corrected_low_height + 3; + + /*Compute size at largest level*/ + size_t width = size_t(corrected_low_width / max_scale); + size_t height = size_t(corrected_low_height / max_scale); + + output = image::Image(width, height, true, T(0.0f)); + output.block(doffset_y, doffset_x, input.Height(), input.Width()) = input; + + return true; +} + +class LaplacianCompositer : public Compositer +{ +public: + LaplacianCompositer(size_t outputWidth, size_t outputHeight, size_t bands) + : Compositer(outputWidth, outputHeight) + , _pyramid_panorama(outputWidth, outputHeight, bands) + , _bands(bands) + { + } + + virtual bool append(const aliceVision::image::Image& color, + const aliceVision::image::Image& inputMask, + const aliceVision::image::Image& inputWeights, size_t offset_x, size_t offset_y) + { + /*Get smalles size*/ + size_t minsize = std::min(color.Height(), color.Width()); + + /* + Look for the smallest scale such that the image is not smaller than the + convolution window size. + minsize / 2^x = 5 + minsize / 5 = 2^x + x = log2(minsize/5) + */ + const float gaussian_filter_size = 5.0f; + size_t optimal_scale = size_t(floor(std::log2(double(minsize) / gaussian_filter_size))); + if(optimal_scale < _bands) + { + ALICEVISION_LOG_ERROR("Decreasing scale !"); + return false; + } + + if(optimal_scale > _bands) + { + _bands = optimal_scale; + _pyramid_panorama.augment(_bands); + } + + size_t new_offset_x, new_offset_y; + aliceVision::image::Image color_pot; + aliceVision::image::Image mask_pot; + aliceVision::image::Image weights_pot; + makeImagePyramidCompatible(color_pot, new_offset_x, new_offset_y, color, offset_x, offset_y, _bands); + makeImagePyramidCompatible(mask_pot, new_offset_x, new_offset_y, inputMask, offset_x, offset_y, _bands); + makeImagePyramidCompatible(weights_pot, new_offset_x, new_offset_y, inputWeights, offset_x, offset_y, _bands); + + aliceVision::image::Image feathered; + feathering(feathered, color_pot, mask_pot); + + /*To log space for hdr*/ + for(int i = 0; i < feathered.Height(); i++) + { + for(int j = 0; j < feathered.Width(); j++) + { + + feathered(i, j).r() = std::log(std::max(1e-8f, feathered(i, j).r())); + feathered(i, j).g() = std::log(std::max(1e-8f, feathered(i, j).g())); + feathered(i, j).b() = std::log(std::max(1e-8f, feathered(i, j).b())); + } + } + + _pyramid_panorama.apply(feathered, mask_pot, weights_pot, new_offset_x, new_offset_y); + + return true; + } + + virtual bool terminate() + { + + _pyramid_panorama.rebuild(_panorama); + + /*Go back to normal space from log space*/ + for(int i = 0; i < _panorama.Height(); i++) + { + for(int j = 0; j < _panorama.Width(); j++) + { + _panorama(i, j).r() = std::exp(_panorama(i, j).r()); + _panorama(i, j).g() = std::exp(_panorama(i, j).g()); + _panorama(i, j).b() = std::exp(_panorama(i, j).b()); + } + } + + return true; + } + +protected: + LaplacianPyramid _pyramid_panorama; + size_t _bands; +}; + +} // namespace aliceVision diff --git a/src/aliceVision/panorama/laplacianPyramid.cpp b/src/aliceVision/panorama/laplacianPyramid.cpp new file mode 100644 index 0000000000..861a0aa0fc --- /dev/null +++ b/src/aliceVision/panorama/laplacianPyramid.cpp @@ -0,0 +1,356 @@ +#include "laplacianPyramid.hpp" + +#include "feathering.hpp" +#include "gaussian.hpp" + +namespace aliceVision +{ + +LaplacianPyramid::LaplacianPyramid(size_t base_width, size_t base_height, size_t max_levels) +{ + + size_t width = base_width; + size_t height = base_height; + + /*Make sure pyramid size can be divided by 2 on each levels*/ + double max_scale = 1.0 / pow(2.0, max_levels - 1); + // width = size_t(ceil(double(width) * max_scale) / max_scale); + // height = size_t(ceil(double(height) * max_scale) / max_scale); + + /*Prepare pyramid*/ + for(int lvl = 0; lvl < max_levels; lvl++) + { + + _levels.push_back( + aliceVision::image::Image(width, height, true, image::RGBfColor(0.0f, 0.0f, 0.0f))); + _weights.push_back(aliceVision::image::Image(width, height, true, 0.0f)); + + height /= 2; + width /= 2; + } +} + +bool LaplacianPyramid::augment(size_t new_max_levels) +{ + + if(new_max_levels <= _levels.size()) + { + return false; + } + + size_t old_max_level = _levels.size(); + + image::Image current_color = _levels[_levels.size() - 1]; + image::Image current_weights = _weights[_weights.size() - 1]; + + _levels[_levels.size() - 1].fill(image::RGBfColor(0.0f, 0.0f, 0.0f)); + _weights[_weights.size() - 1].fill(0.0f); + + image::Image current_mask(current_color.Width(), current_color.Height(), true, 0); + image::Image current_color_feathered(current_color.Width(), current_color.Height()); + + for(int i = 0; i < current_color.Height(); i++) + { + for(int j = 0; j < current_color.Width(); j++) + { + if(current_weights(i, j) < 1e-6) + { + current_color(i, j) = image::RGBfColor(0.0); + continue; + } + + current_color(i, j).r() = current_color(i, j).r() / current_weights(i, j); + current_color(i, j).g() = current_color(i, j).g() / current_weights(i, j); + current_color(i, j).b() = current_color(i, j).b() / current_weights(i, j); + current_mask(i, j) = 255; + } + } + + feathering(current_color_feathered, current_color, current_mask); + current_color = current_color_feathered; + + for(int l = old_max_level; l < new_max_levels; l++) + { + + _levels.emplace_back(_levels[l - 1].Width() / 2, _levels[l - 1].Height() / 2, true, + image::RGBfColor(0.0f, 0.0f, 0.0f)); + _weights.emplace_back(_weights[l - 1].Width() / 2, _weights[l - 1].Height() / 2, true, 0.0f); + } + + int width = current_color.Width(); + int height = current_color.Height(); + image::Image next_color; + image::Image next_weights; + + for(int l = old_max_level - 1; l < new_max_levels - 1; l++) + { + aliceVision::image::Image buf(width, height); + aliceVision::image::Image buf2(width, height); + aliceVision::image::Image bufw(width, height); + + next_color = aliceVision::image::Image(width / 2, height / 2); + next_weights = aliceVision::image::Image(width / 2, height / 2); + + convolveGaussian5x5(buf, current_color); + downscale(next_color, buf); + + convolveGaussian5x5(bufw, current_weights); + downscale(next_weights, bufw); + + upscale(buf, next_color); + convolveGaussian5x5(buf2, buf); + + for(int i = 0; i < buf2.Height(); i++) + { + for(int j = 0; j < buf2.Width(); j++) + { + buf2(i, j) *= 4.0f; + } + } + + substract(current_color, current_color, buf2); + + merge(current_color, current_weights, l, 0, 0); + + current_color = next_color; + current_weights = next_weights; + width /= 2; + height /= 2; + } + + merge(current_color, current_weights, _levels.size() - 1, 0, 0); + + return true; +} + +bool LaplacianPyramid::apply(const aliceVision::image::Image& source, + const aliceVision::image::Image& mask, + const aliceVision::image::Image& weights, size_t offset_x, size_t offset_y) +{ + + int width = source.Width(); + int height = source.Height(); + + /* Convert mask to alpha layer */ + image::Image mask_float(width, height); + for(int i = 0; i < height; i++) + { + for(int j = 0; j < width; j++) + { + if(mask(i, j)) + { + mask_float(i, j) = 1.0f; + } + else + { + mask_float(i, j) = 0.0f; + } + } + } + + image::Image current_color = source; + image::Image next_color; + image::Image current_weights = weights; + image::Image next_weights; + image::Image current_mask = mask_float; + image::Image next_mask; + + for(int l = 0; l < _levels.size() - 1; l++) + { + aliceVision::image::Image buf_masked(width, height); + aliceVision::image::Image buf(width, height); + aliceVision::image::Image buf2(width, height); + aliceVision::image::Image buf_float(width, height); + + next_color = aliceVision::image::Image(width / 2, height / 2); + next_weights = aliceVision::image::Image(width / 2, height / 2); + next_mask = aliceVision::image::Image(width / 2, height / 2); + + /*Apply mask to content before convolution*/ + for(int i = 0; i < current_color.Height(); i++) + { + for(int j = 0; j < current_color.Width(); j++) + { + if(std::abs(current_mask(i, j)) > 1e-6) + { + buf_masked(i, j) = current_color(i, j); + } + else + { + buf_masked(i, j).r() = 0.0f; + buf_masked(i, j).g() = 0.0f; + buf_masked(i, j).b() = 0.0f; + current_weights(i, j) = 0.0f; + } + } + } + + convolveGaussian5x5(buf, buf_masked); + convolveGaussian5x5(buf_float, current_mask); + + /* + Normalize given mask + */ + for(int i = 0; i < current_color.Height(); i++) + { + for(int j = 0; j < current_color.Width(); j++) + { + + float m = buf_float(i, j); + + if(std::abs(m) > 1e-6) + { + buf(i, j).r() = buf(i, j).r() / m; + buf(i, j).g() = buf(i, j).g() / m; + buf(i, j).b() = buf(i, j).b() / m; + buf_float(i, j) = 1.0f; + } + else + { + buf(i, j).r() = 0.0f; + buf(i, j).g() = 0.0f; + buf(i, j).b() = 0.0f; + buf_float(i, j) = 0.0f; + } + } + } + + downscale(next_color, buf); + downscale(next_mask, buf_float); + + upscale(buf, next_color); + convolveGaussian5x5(buf2, buf); + + for(int i = 0; i < buf2.Height(); i++) + { + for(int j = 0; j < buf2.Width(); j++) + { + buf2(i, j) *= 4.0f; + } + } + + substract(current_color, current_color, buf2); + + convolveGaussian5x5(buf_float, current_weights); + downscale(next_weights, buf_float); + + merge(current_color, current_weights, l, offset_x, offset_y); + + current_color = next_color; + current_weights = next_weights; + current_mask = next_mask; + + width /= 2; + height /= 2; + offset_x /= 2; + offset_y /= 2; + } + + merge(current_color, current_weights, _levels.size() - 1, offset_x, offset_y); + + return true; +} + +bool LaplacianPyramid::merge(const aliceVision::image::Image& oimg, + const aliceVision::image::Image& oweight, size_t level, size_t offset_x, + size_t offset_y) +{ + + image::Image& img = _levels[level]; + image::Image& weight = _weights[level]; + + for(int i = 0; i < oimg.Height(); i++) + { + + int di = i + offset_y; + if(di >= img.Height()) + continue; + + for(int j = 0; j < oimg.Width(); j++) + { + + int dj = j + offset_x; + if(dj >= weight.Width()) + { + dj = dj - weight.Width(); + } + + img(di, dj).r() += oimg(i, j).r() * oweight(i, j); + img(di, dj).g() += oimg(i, j).g() * oweight(i, j); + img(di, dj).b() += oimg(i, j).b() * oweight(i, j); + weight(di, dj) += oweight(i, j); + } + } + + return true; +} + +bool LaplacianPyramid::rebuild(image::Image& output) +{ + + for(int l = 0; l < _levels.size(); l++) + { + for(int i = 0; i < _levels[l].Height(); i++) + { + for(int j = 0; j < _levels[l].Width(); j++) + { + if(_weights[l](i, j) < 1e-6) + { + _levels[l](i, j) = image::RGBfColor(0.0); + continue; + } + + _levels[l](i, j).r() = _levels[l](i, j).r() / _weights[l](i, j); + _levels[l](i, j).g() = _levels[l](i, j).g() / _weights[l](i, j); + _levels[l](i, j).b() = _levels[l](i, j).b() / _weights[l](i, j); + } + } + } + + removeNegativeValues(_levels[_levels.size() - 1]); + + for(int l = _levels.size() - 2; l >= 0; l--) + { + + aliceVision::image::Image buf(_levels[l].Width(), _levels[l].Height()); + aliceVision::image::Image buf2(_levels[l].Width(), _levels[l].Height()); + + upscale(buf, _levels[l + 1]); + convolveGaussian5x5(buf2, buf, true); + + for(int i = 0; i < buf2.Height(); i++) + { + for(int j = 0; j < buf2.Width(); j++) + { + buf2(i, j) *= 4.0f; + } + } + + addition(_levels[l], _levels[l], buf2); + removeNegativeValues(_levels[l]); + } + + // Write output to RGBA + for(int i = 0; i < output.Height(); i++) + { + for(int j = 0; j < output.Width(); j++) + { + output(i, j).r() = _levels[0](i, j).r(); + output(i, j).g() = _levels[0](i, j).g(); + output(i, j).b() = _levels[0](i, j).b(); + + if(_weights[0](i, j) < 1e-6) + { + output(i, j).a() = 0.0f; + } + else + { + output(i, j).a() = 1.0f; + } + } + } + + return true; +} + +} // namespace aliceVision \ No newline at end of file diff --git a/src/aliceVision/panorama/laplacianPyramid.hpp b/src/aliceVision/panorama/laplacianPyramid.hpp new file mode 100644 index 0000000000..87b86fafdd --- /dev/null +++ b/src/aliceVision/panorama/laplacianPyramid.hpp @@ -0,0 +1,25 @@ +#pragma once + +#include "imageOps.hpp" + +namespace aliceVision +{ + +class LaplacianPyramid +{ +public: + LaplacianPyramid(size_t base_width, size_t base_height, size_t max_levels); + bool augment(size_t new_max_levels); + bool apply(const aliceVision::image::Image& source, + const aliceVision::image::Image& mask, const aliceVision::image::Image& weights, + size_t offset_x, size_t offset_y); + bool merge(const aliceVision::image::Image& oimg, const aliceVision::image::Image& oweight, + size_t level, size_t offset_x, size_t offset_y); + bool rebuild(image::Image& output); + +private: + std::vector> _levels; + std::vector> _weights; +}; + +} // namespace aliceVision \ No newline at end of file diff --git a/src/aliceVision/panorama/remapBbox.cpp b/src/aliceVision/panorama/remapBbox.cpp index a0234047d0..fe2d967c64 100644 --- a/src/aliceVision/panorama/remapBbox.cpp +++ b/src/aliceVision/panorama/remapBbox.cpp @@ -1,46 +1,57 @@ -#pragma once - +#pragma once #include "remapBbox.hpp" #include "sphericalMapping.hpp" -namespace aliceVision { +namespace aliceVision +{ + +bool isPoleInTriangle(const Vec3& pt1, const Vec3& pt2, const Vec3& pt3) +{ -bool isPoleInTriangle(const Vec3 & pt1, const Vec3 & pt2, const Vec3 & pt3) { - - double a = (pt2.x()*pt3.z() - pt3.x()*pt2.z())/(pt1.x()*pt2.z() - pt1.x()*pt3.z() - pt2.x()*pt1.z() + pt2.x()*pt3.z() + pt3.x()*pt1.z() - pt3.x()*pt2.z()); - double b = (-pt1.x()*pt3.z() + pt3.x()*pt1.z())/(pt1.x()*pt2.z() - pt1.x()*pt3.z() - pt2.x()*pt1.z() + pt2.x()*pt3.z() + pt3.x()*pt1.z() - pt3.x()*pt2.z()); + double a = (pt2.x() * pt3.z() - pt3.x() * pt2.z()) / (pt1.x() * pt2.z() - pt1.x() * pt3.z() - pt2.x() * pt1.z() + + pt2.x() * pt3.z() + pt3.x() * pt1.z() - pt3.x() * pt2.z()); + double b = (-pt1.x() * pt3.z() + pt3.x() * pt1.z()) / (pt1.x() * pt2.z() - pt1.x() * pt3.z() - pt2.x() * pt1.z() + + pt2.x() * pt3.z() + pt3.x() * pt1.z() - pt3.x() * pt2.z()); double c = 1.0 - a - b; - if (a < 0.0 || a > 1.0) return false; - if (b < 0.0 || b > 1.0) return false; - if (c < 0.0 || c > 1.0) return false; - + if(a < 0.0 || a > 1.0) + return false; + if(b < 0.0 || b > 1.0) + return false; + if(c < 0.0 || c > 1.0) + return false; + return true; } -bool crossHorizontalLoop(const Vec3 & pt1, const Vec3 & pt2) { +bool crossHorizontalLoop(const Vec3& pt1, const Vec3& pt2) +{ Vec3 direction = pt2 - pt1; /*Vertical line*/ - if (std::abs(direction(0)) < 1e-12) { - return false; + if(std::abs(direction(0)) < 1e-12) + { + return false; } - double t = - pt1(0) / direction(0); + double t = -pt1(0) / direction(0); Vec3 cross = pt1 + direction * t; - if (t >= 0.0 && t <= 1.0) { - if (cross(2) < 0.0) { - return true; - } + if(t >= 0.0 && t <= 1.0) + { + if(cross(2) < 0.0) + { + return true; + } } return false; } -Vec3 getExtremaY(const Vec3 & pt1, const Vec3 & pt2) { - +Vec3 getExtremaY(const Vec3& pt1, const Vec3& pt2) +{ + Vec3 delta = pt2 - pt1; double dx = delta(0); double dy = delta(1); @@ -49,137 +60,152 @@ Vec3 getExtremaY(const Vec3 & pt1, const Vec3 & pt2) { double sy = pt1(1); double sz = pt1(2); - double ot_y = -(dx*sx*sy - (dy*sx)*(dy*sx) - (dy*sz)*(dy*sz) + dz*sy*sz)/(dx*dx*sy - dx*dy*sx - dy*dz*sz + dz*dz*sy); + double ot_y = -(dx * sx * sy - (dy * sx) * (dy * sx) - (dy * sz) * (dy * sz) + dz * sy * sz) / + (dx * dx * sy - dx * dy * sx - dy * dz * sz + dz * dz * sy); Vec3 pt_extrema = pt1 + ot_y * delta; return pt_extrema.normalized(); } -bool computeCoarseBB_Equidistant(BoundingBox & coarse_bbox, const std::pair & panoramaSize, const geometry::Pose3 & pose, const aliceVision::camera::IntrinsicBase & intrinsics) { - - const aliceVision::camera::EquiDistant & cam = dynamic_cast(intrinsics); - +bool computeCoarseBB_Equidistant(BoundingBox& coarse_bbox, const std::pair& panoramaSize, + const geometry::Pose3& pose, const aliceVision::camera::IntrinsicBase& intrinsics) +{ + + const aliceVision::camera::EquiDistant& cam = dynamic_cast(intrinsics); bool loop = false; std::vector vec_bool(panoramaSize.second, false); - for (int i = 0; i < panoramaSize.second; i++) { - - { - Vec3 ray = SphericalMapping::fromEquirectangular(Vec2(0, i), panoramaSize.first, panoramaSize.second); - - /** - * Check that this ray should be visible. - * This test is camera type dependent - */ - Vec3 transformedRay = pose(ray); - if (!intrinsics.isVisibleRay(transformedRay)) { - continue; - } - - /** - * Project this ray to camera pixel coordinates - */ - const Vec2 pix_disto = intrinsics.project(pose, ray, true); - - /** - * Ignore invalid coordinates - */ - if (!intrinsics.isVisible(pix_disto)) { - continue; - } - } - - { - Vec3 ray = SphericalMapping::fromEquirectangular(Vec2(panoramaSize.first - 1, i), panoramaSize.first, panoramaSize.second); - - /** - * Check that this ray should be visible. - * This test is camera type dependent - */ - Vec3 transformedRay = pose(ray); - if (!intrinsics.isVisibleRay(transformedRay)) { - continue; + for(int i = 0; i < panoramaSize.second; i++) + { + + { + Vec3 ray = SphericalMapping::fromEquirectangular(Vec2(0, i), panoramaSize.first, panoramaSize.second); + + /** + * Check that this ray should be visible. + * This test is camera type dependent + */ + Vec3 transformedRay = pose(ray); + if(!intrinsics.isVisibleRay(transformedRay)) + { + continue; + } + + /** + * Project this ray to camera pixel coordinates + */ + const Vec2 pix_disto = intrinsics.project(pose, ray, true); + + /** + * Ignore invalid coordinates + */ + if(!intrinsics.isVisible(pix_disto)) + { + continue; + } } - /** - * Project this ray to camera pixel coordinates - */ - const Vec2 pix_disto = intrinsics.project(pose, ray, true); - - /** - * Ignore invalid coordinates - */ - if (!intrinsics.isVisible(pix_disto)) { - continue; + { + Vec3 ray = SphericalMapping::fromEquirectangular(Vec2(panoramaSize.first - 1, i), panoramaSize.first, + panoramaSize.second); + + /** + * Check that this ray should be visible. + * This test is camera type dependent + */ + Vec3 transformedRay = pose(ray); + if(!intrinsics.isVisibleRay(transformedRay)) + { + continue; + } + + /** + * Project this ray to camera pixel coordinates + */ + const Vec2 pix_disto = intrinsics.project(pose, ray, true); + + /** + * Ignore invalid coordinates + */ + if(!intrinsics.isVisible(pix_disto)) + { + continue; + } + + vec_bool[i] = true; + loop = true; } - - vec_bool[i] = true; - loop = true; - } } - if (vec_bool[0] || vec_bool[panoramaSize.second - 1]) { - loop = false; + if(vec_bool[0] || vec_bool[panoramaSize.second - 1]) + { + loop = false; } - if (!loop) { - coarse_bbox.left = 0; - coarse_bbox.top = 0; - coarse_bbox.width = panoramaSize.first; - coarse_bbox.height = panoramaSize.second; - return true; + if(!loop) + { + coarse_bbox.left = 0; + coarse_bbox.top = 0; + coarse_bbox.width = panoramaSize.first; + coarse_bbox.height = panoramaSize.second; + return true; } int last_x = 0; - for (int x = panoramaSize.first - 1; x >= 0; x--) { - - size_t count = 0; - - for (int i = 0; i < panoramaSize.second; i++) { - - if (vec_bool[i] == false) { - continue; - } - - Vec3 ray = SphericalMapping::fromEquirectangular(Vec2(x, i), panoramaSize.first, panoramaSize.second); - - /** - * Check that this ray should be visible. - * This test is camera type dependent - */ - Vec3 transformedRay = pose(ray); - if (!intrinsics.isVisibleRay(transformedRay)) { - vec_bool[i] = false; - continue; + for(int x = panoramaSize.first - 1; x >= 0; x--) + { + + size_t count = 0; + + for(int i = 0; i < panoramaSize.second; i++) + { + + if(vec_bool[i] == false) + { + continue; + } + + Vec3 ray = SphericalMapping::fromEquirectangular(Vec2(x, i), panoramaSize.first, panoramaSize.second); + + /** + * Check that this ray should be visible. + * This test is camera type dependent + */ + Vec3 transformedRay = pose(ray); + if(!intrinsics.isVisibleRay(transformedRay)) + { + vec_bool[i] = false; + continue; + } + + /** + * Project this ray to camera pixel coordinates + */ + const Vec2 pix_disto = intrinsics.project(pose, ray, true); + + /** + * Ignore invalid coordinates + */ + if(!intrinsics.isVisible(pix_disto)) + { + vec_bool[i] = false; + continue; + } + + count++; } - /** - * Project this ray to camera pixel coordinates - */ - const Vec2 pix_disto = intrinsics.project(pose, ray, true); - - /** - * Ignore invalid coordinates - */ - if (!intrinsics.isVisible(pix_disto)) { - vec_bool[i] = false; - continue; + if(count == 0) + { + break; } - - count++; - } - if (count == 0) { - break; - } - - last_x = x; + last_x = x; } - coarse_bbox.left = last_x; coarse_bbox.top = 0; coarse_bbox.width = panoramaSize.first; @@ -188,7 +214,9 @@ bool computeCoarseBB_Equidistant(BoundingBox & coarse_bbox, const std::pair & panoramaSize, const geometry::Pose3 & pose, const aliceVision::camera::IntrinsicBase & intrinsics) { +bool computeCoarseBB_Pinhole(BoundingBox& coarse_bbox, const std::pair& panoramaSize, + const geometry::Pose3& pose, const aliceVision::camera::IntrinsicBase& intrinsics) +{ int bbox_left, bbox_top; int bbox_right, bbox_bottom; @@ -197,62 +225,58 @@ bool computeCoarseBB_Pinhole(BoundingBox & coarse_bbox, const std::pair 0) { - //Lower pole - bbox_bottom = panoramaSize.second - 1; - } - else { - //upper pole - bbox_top = 0; - } + + if(pole) + { + Vec3 normal = (rotated_pts[1] - rotated_pts[0]).cross(rotated_pts[3] - rotated_pts[0]); + if(normal(1) > 0) + { + // Lower pole + bbox_bottom = panoramaSize.second - 1; + } + else + { + // upper pole + bbox_top = 0; + } } bbox_height = bbox_bottom - bbox_top + 1; - /*Check if we cross the horizontal loop*/ bool crossH = false; - for (int i = 0; i < 8; i++) { - int i2 = (i + 1) % 8; + for(int i = 0; i < 8; i++) + { + int i2 = (i + 1) % 8; - bool cross = crossHorizontalLoop(rotated_pts[i], rotated_pts[i2]); - crossH |= cross; + bool cross = crossHorizontalLoop(rotated_pts[i], rotated_pts[i2]); + crossH |= cross; } - if (pole) { - /*Easy : if we cross the pole, the width is full*/ - bbox_left = 0; - bbox_right = panoramaSize.first - 1; - bbox_width = bbox_right - bbox_left + 1; + if(pole) + { + /*Easy : if we cross the pole, the width is full*/ + bbox_left = 0; + bbox_right = panoramaSize.first - 1; + bbox_width = bbox_right - bbox_left + 1; } - else if (crossH) { - - int first_cross = 0; - for (int i = 0; i < 8; i++) { - int i2 = (i + 1) % 8; - bool cross = crossHorizontalLoop(rotated_pts[i], rotated_pts[i2]); - if (cross) { - first_cross = i; - break; + else if(crossH) + { + + int first_cross = 0; + for(int i = 0; i < 8; i++) + { + int i2 = (i + 1) % 8; + bool cross = crossHorizontalLoop(rotated_pts[i], rotated_pts[i2]); + if(cross) + { + first_cross = i; + break; + } } - } - - bbox_left = panoramaSize.first - 1; - bbox_right = 0; - bool is_right = true; - for (int index = 0; index < 8; index++) { - - int i = (index + first_cross) % 8; - int i2 = (i + 1) % 8; - - Vec2 res_1 = SphericalMapping::toEquirectangular(rotated_pts[i], panoramaSize.first, panoramaSize.second); - Vec2 res_2 = SphericalMapping::toEquirectangular(rotated_pts[i2], panoramaSize.first, panoramaSize.second); - /*[----right //// left-----]*/ - bool cross = crossHorizontalLoop(rotated_pts[i], rotated_pts[i2]); - if (cross) { - if (res_1(0) > res_2(0)) { /*[----res2 //// res1----]*/ - bbox_left = std::min(int(res_1(0)), bbox_left); - bbox_right = std::max(int(res_2(0)), bbox_right); - is_right = true; - } - else { /*[----res1 //// res2----]*/ - bbox_left = std::min(int(res_2(0)), bbox_left); - bbox_right = std::max(int(res_1(0)), bbox_right); - is_right = false; - } - } - else { - if (is_right) { - bbox_right = std::max(int(res_1(0)), bbox_right); - bbox_right = std::max(int(res_2(0)), bbox_right); - } - else { - bbox_left = std::min(int(res_1(0)), bbox_left); - bbox_left = std::min(int(res_2(0)), bbox_left); - } + bbox_left = panoramaSize.first - 1; + bbox_right = 0; + bool is_right = true; + for(int index = 0; index < 8; index++) + { + + int i = (index + first_cross) % 8; + int i2 = (i + 1) % 8; + + Vec2 res_1 = SphericalMapping::toEquirectangular(rotated_pts[i], panoramaSize.first, panoramaSize.second); + Vec2 res_2 = SphericalMapping::toEquirectangular(rotated_pts[i2], panoramaSize.first, panoramaSize.second); + + /*[----right //// left-----]*/ + bool cross = crossHorizontalLoop(rotated_pts[i], rotated_pts[i2]); + if(cross) + { + if(res_1(0) > res_2(0)) + { /*[----res2 //// res1----]*/ + bbox_left = std::min(int(res_1(0)), bbox_left); + bbox_right = std::max(int(res_2(0)), bbox_right); + is_right = true; + } + else + { /*[----res1 //// res2----]*/ + bbox_left = std::min(int(res_2(0)), bbox_left); + bbox_right = std::max(int(res_1(0)), bbox_right); + is_right = false; + } + } + else + { + if(is_right) + { + bbox_right = std::max(int(res_1(0)), bbox_right); + bbox_right = std::max(int(res_2(0)), bbox_right); + } + else + { + bbox_left = std::min(int(res_1(0)), bbox_left); + bbox_left = std::min(int(res_2(0)), bbox_left); + } + } } - } - bbox_width = bbox_right + (panoramaSize.first - bbox_left); + bbox_width = bbox_right + (panoramaSize.first - bbox_left); } - else { - /*horizontal default solution : no border crossing, no pole*/ - bbox_left = panoramaSize.first; - bbox_right = 0; - for (int i = 0; i < 8; i++) { - Vec2 res = SphericalMapping::toEquirectangular(rotated_pts[i], panoramaSize.first, panoramaSize.second); - bbox_left = std::min(int(floor(res(0))), bbox_left); - bbox_right = std::max(int(ceil(res(0))), bbox_right); - } - bbox_width = bbox_right - bbox_left + 1; + else + { + /*horizontal default solution : no border crossing, no pole*/ + bbox_left = panoramaSize.first; + bbox_right = 0; + for(int i = 0; i < 8; i++) + { + Vec2 res = SphericalMapping::toEquirectangular(rotated_pts[i], panoramaSize.first, panoramaSize.second); + bbox_left = std::min(int(floor(res(0))), bbox_left); + bbox_right = std::max(int(ceil(res(0))), bbox_right); + } + bbox_width = bbox_right - bbox_left + 1; } /*Assign solution to result*/ @@ -363,23 +402,26 @@ bool computeCoarseBB_Pinhole(BoundingBox & coarse_bbox, const std::pair & panoramaSize, const geometry::Pose3 & pose, const aliceVision::camera::IntrinsicBase & intrinsics) { +bool computeCoarseBB(BoundingBox& coarse_bbox, const std::pair& panoramaSize, const geometry::Pose3& pose, + const aliceVision::camera::IntrinsicBase& intrinsics) +{ bool ret = true; - if (isPinhole(intrinsics.getType())) { + if(isPinhole(intrinsics.getType())) + { ret = computeCoarseBB_Pinhole(coarse_bbox, panoramaSize, pose, intrinsics); } - else if (isEquidistant(intrinsics.getType())) { + else if(isEquidistant(intrinsics.getType())) + { ret = computeCoarseBB_Equidistant(coarse_bbox, panoramaSize, pose, intrinsics); } - else { + else + { coarse_bbox.left = 0; coarse_bbox.top = 0; coarse_bbox.width = panoramaSize.first; @@ -390,4 +432,4 @@ bool computeCoarseBB(BoundingBox & coarse_bbox, const std::pair & pano return ret; } -} \ No newline at end of file +} // namespace aliceVision \ No newline at end of file diff --git a/src/aliceVision/panorama/remapBbox.hpp b/src/aliceVision/panorama/remapBbox.hpp index ad6594f95f..f224ae3a11 100644 --- a/src/aliceVision/panorama/remapBbox.hpp +++ b/src/aliceVision/panorama/remapBbox.hpp @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include #include @@ -6,8 +6,10 @@ #include "boundingBox.hpp" -namespace aliceVision { +namespace aliceVision +{ -bool computeCoarseBB(BoundingBox & coarse_bbox, const std::pair & panoramaSize, const geometry::Pose3 & pose, const aliceVision::camera::IntrinsicBase & intrinsics); +bool computeCoarseBB(BoundingBox& coarse_bbox, const std::pair& panoramaSize, const geometry::Pose3& pose, + const aliceVision::camera::IntrinsicBase& intrinsics); } \ No newline at end of file diff --git a/src/aliceVision/panorama/seams.cpp b/src/aliceVision/panorama/seams.cpp new file mode 100644 index 0000000000..bcb4f7c333 --- /dev/null +++ b/src/aliceVision/panorama/seams.cpp @@ -0,0 +1,320 @@ +#include "seams.hpp" + +#include "gaussian.hpp" +#include "imageOps.hpp" + +namespace aliceVision +{ + +void drawBorders(aliceVision::image::Image& inout, aliceVision::image::Image& mask, + size_t offset_x, size_t offset_y) +{ + + for(int i = 0; i < mask.Height(); i++) + { + int j = 0; + int di = i + offset_y; + int dj = j + offset_x; + if(dj >= inout.Width()) + { + dj = dj - inout.Width(); + } + + if(mask(i, j)) + { + inout(di, dj) = image::RGBAfColor(0.0f, 1.0f, 0.0f, 1.0f); + } + } + + for(int i = 0; i < mask.Height(); i++) + { + int j = mask.Width() - 1; + int di = i + offset_y; + int dj = j + offset_x; + if(dj >= inout.Width()) + { + dj = dj - inout.Width(); + } + + if(mask(i, j)) + { + inout(di, dj) = image::RGBAfColor(0.0f, 1.0f, 0.0f, 1.0f); + } + } + + for(int j = 0; j < mask.Width(); j++) + { + int i = 0; + int di = i + offset_y; + int dj = j + offset_x; + if(dj >= inout.Width()) + { + dj = dj - inout.Width(); + } + + if(mask(i, j)) + { + inout(di, dj) = image::RGBAfColor(0.0f, 1.0f, 0.0f, 1.0f); + } + } + + for(int j = 0; j < mask.Width(); j++) + { + int i = mask.Height() - 1; + int di = i + offset_y; + int dj = j + offset_x; + if(dj >= inout.Width()) + { + dj = dj - inout.Width(); + } + + if(mask(i, j)) + { + inout(di, dj) = image::RGBAfColor(0.0f, 1.0f, 0.0f, 1.0f); + } + } + + for(int i = 1; i < mask.Height() - 1; i++) + { + + int di = i + offset_y; + + for(int j = 1; j < mask.Width() - 1; j++) + { + + int dj = j + offset_x; + if(dj >= inout.Width()) + { + dj = dj - inout.Width(); + } + + if(!mask(i, j)) + continue; + + unsigned char others = true; + others &= mask(i - 1, j - 1); + others &= mask(i - 1, j + 1); + others &= mask(i, j - 1); + others &= mask(i, j + 1); + others &= mask(i + 1, j - 1); + others &= mask(i + 1, j + 1); + if(others) + continue; + + inout(di, dj) = image::RGBAfColor(0.0f, 1.0f, 0.0f, 1.0f); + } + } +} + +void drawSeams(aliceVision::image::Image& inout, aliceVision::image::Image& labels) +{ + + for(int i = 1; i < labels.Height() - 1; i++) + { + + for(int j = 1; j < labels.Width() - 1; j++) + { + + IndexT label = labels(i, j); + IndexT same = true; + + same &= (labels(i - 1, j - 1) == label); + same &= (labels(i - 1, j + 1) == label); + same &= (labels(i, j - 1) == label); + same &= (labels(i, j + 1) == label); + same &= (labels(i + 1, j - 1) == label); + same &= (labels(i + 1, j + 1) == label); + + if(same) + { + continue; + } + + inout(i, j) = image::RGBAfColor(1.0f, 0.0f, 0.0f, 1.0f); + } + } +} + +bool WTASeams::append(const aliceVision::image::Image& inputMask, + const aliceVision::image::Image& inputWeights, IndexT currentIndex, size_t offset_x, + size_t offset_y) +{ + if(inputMask.size() != inputWeights.size()) + { + return false; + } + + for(int i = 0; i < inputMask.Height(); i++) + { + + int di = i + offset_y; + + for(int j = 0; j < inputMask.Width(); j++) + { + + if(!inputMask(i, j)) + { + continue; + } + + int dj = j + offset_x; + if(dj >= _weights.Width()) + { + dj = dj - _weights.Width(); + } + + if(inputWeights(i, j) > _weights(di, dj)) + { + _labels(di, dj) = currentIndex; + _weights(di, dj) = inputWeights(i, j); + } + } + } + + return true; +} + +void HierarchicalGraphcutSeams::setOriginalLabels(const image::Image& labels) +{ + + /* + First of all, Propagate label to all levels + */ + image::Image current_label = labels; + + for(int l = 1; l <= _levelOfInterest; l++) + { + + aliceVision::image::Image next_label(current_label.Width() / 2, current_label.Height() / 2); + + for(int i = 0; i < next_label.Height(); i++) + { + int di = i * 2; + + for(int j = 0; j < next_label.Width(); j++) + { + int dj = j * 2; + + next_label(i, j) = current_label(di, dj); + } + } + + current_label = next_label; + } + + _graphcut->setOriginalLabels(current_label); +} + +bool HierarchicalGraphcutSeams::append(const aliceVision::image::Image& input, + const aliceVision::image::Image& inputMask, IndexT currentIndex, + size_t offset_x, size_t offset_y) +{ + image::Image current_color = input; + image::Image current_mask = inputMask; + + for(int l = 1; l <= _levelOfInterest; l++) + { + + aliceVision::image::Image buf(current_color.Width(), current_color.Height()); + aliceVision::image::Image next_color(current_color.Width() / 2, current_color.Height() / 2); + aliceVision::image::Image next_mask(current_color.Width() / 2, current_color.Height() / 2); + + convolveGaussian5x5(buf, current_color); + downscale(next_color, buf); + + for(int i = 0; i < next_mask.Height(); i++) + { + int di = i * 2; + + for(int j = 0; j < next_mask.Width(); j++) + { + int dj = j * 2; + + if(current_mask(di, dj) && current_mask(di, dj + 1) && current_mask(di + 1, dj) && + current_mask(di + 1, dj + 1)) + { + next_mask(i, j) = 255; + } + else + { + next_mask(i, j) = 0; + } + } + } + + current_color = next_color; + current_mask = next_mask; + offset_x /= 2; + offset_y /= 2; + } + + return _graphcut->append(current_color, current_mask, currentIndex, offset_x, offset_y); +} + +bool HierarchicalGraphcutSeams::process() +{ + + if(!_graphcut->process()) + { + return false; + } + + image::Image current_labels = _graphcut->getLabels(); + + for(int l = _levelOfInterest - 1; l >= 0; l--) + { + + int nw = current_labels.Width() * 2; + int nh = current_labels.Height() * 2; + if(l == 0) + { + nw = _outputWidth; + nh = _outputHeight; + } + + aliceVision::image::Image next_label(nw, nh); + for(int i = 0; i < nh; i++) + { + int hi = i / 2; + + for(int j = 0; j < nw; j++) + { + int hj = j / 2; + + next_label(i, j) = current_labels(hi, hj); + } + } + + current_labels = next_label; + } + + _labels = current_labels; + + return true; +} + +void getMaskFromLabels(aliceVision::image::Image & mask, aliceVision::image::Image & labels, IndexT index, size_t offset_x, size_t offset_y) { + + for (int i = 0; i < mask.Height(); i++) { + + int di = i + offset_y; + + for (int j = 0; j < mask.Width(); j++) { + + int dj = j + offset_x; + if (dj >= labels.Width()) { + dj = dj - labels.Width(); + } + + + if (labels(di, dj) == index) { + mask(i, j) = 1.0f; + } + else { + mask(i, j) = 0.0f; + } + } + } +} + +} // namespace aliceVision \ No newline at end of file diff --git a/src/aliceVision/panorama/seams.hpp b/src/aliceVision/panorama/seams.hpp new file mode 100644 index 0000000000..57ab7e958b --- /dev/null +++ b/src/aliceVision/panorama/seams.hpp @@ -0,0 +1,78 @@ +#pragma once + +#include +#include + +#include "graphcut.hpp" + +namespace aliceVision +{ + +void drawBorders(aliceVision::image::Image& inout, aliceVision::image::Image& mask, + size_t offset_x, size_t offset_y); +void drawSeams(aliceVision::image::Image& inout, aliceVision::image::Image& labels); + +void getMaskFromLabels(aliceVision::image::Image & mask, aliceVision::image::Image & labels, IndexT index, size_t offset_x, size_t offset_y); + +class WTASeams +{ +public: + WTASeams(size_t outputWidth, size_t outputHeight) + : _weights(outputWidth, outputHeight, true, 0.0f) + , _labels(outputWidth, outputHeight, true, 255) + { + } + + virtual ~WTASeams() = default; + + virtual bool append(const aliceVision::image::Image& inputMask, + const aliceVision::image::Image& inputWeights, IndexT currentIndex, size_t offset_x, + size_t offset_y); + + const image::Image& getLabels() { return _labels; } + +private: + image::Image _weights; + image::Image _labels; +}; + +class HierarchicalGraphcutSeams +{ +public: + HierarchicalGraphcutSeams(size_t outputWidth, size_t outputHeight, size_t levelOfInterest) + : _outputWidth(outputWidth) + , _outputHeight(outputHeight) + , _levelOfInterest(levelOfInterest) + , _labels(outputWidth, outputHeight, true, UndefinedIndexT) + { + + double scale = 1.0 / pow(2.0, levelOfInterest); + size_t width = size_t(floor(double(outputWidth) * scale)); + size_t height = size_t(floor(double(outputHeight) * scale)); + + _graphcut = std::unique_ptr(new GraphcutSeams(width, height)); + } + + virtual ~HierarchicalGraphcutSeams() = default; + + void setOriginalLabels(const image::Image& labels); + + void setMaximalDistance(int distance) { _graphcut->setMaximalDistance(distance); } + + virtual bool append(const aliceVision::image::Image& input, + const aliceVision::image::Image& inputMask, IndexT currentIndex, size_t offset_x, + size_t offset_y); + + bool process(); + + const image::Image& getLabels() { return _labels; } + +private: + std::unique_ptr _graphcut; + image::Image _labels; + size_t _levelOfInterest; + size_t _outputWidth; + size_t _outputHeight; +}; + +} // namespace aliceVision \ No newline at end of file diff --git a/src/aliceVision/panorama/sphericalMapping.cpp b/src/aliceVision/panorama/sphericalMapping.cpp index f499f83b19..a93439c0e9 100644 --- a/src/aliceVision/panorama/sphericalMapping.cpp +++ b/src/aliceVision/panorama/sphericalMapping.cpp @@ -1,6 +1,6 @@ #include "sphericalMapping.hpp" -namespace aliceVision +namespace aliceVision { namespace SphericalMapping @@ -12,9 +12,9 @@ namespace SphericalMapping * @param height number of pixels used to represent latitude * @return spherical coordinates */ -Vec3 fromEquirectangular(const Vec2 & equirectangular, int width, int height) +Vec3 fromEquirectangular(const Vec2& equirectangular, int width, int height) { - const double latitude = (equirectangular(1) / double(height)) * M_PI - M_PI_2; + const double latitude = (equirectangular(1) / double(height)) * M_PI - M_PI_2; const double longitude = ((equirectangular(0) / double(width)) * 2.0 * M_PI) - M_PI; const double Px = cos(latitude) * sin(longitude); @@ -31,16 +31,17 @@ Vec3 fromEquirectangular(const Vec2 & equirectangular, int width, int height) * @param height number of pixels used to represent latitude * @return equirectangular coordinates */ -Vec2 toEquirectangular(const Vec3 & spherical, int width, int height) { +Vec2 toEquirectangular(const Vec3& spherical, int width, int height) +{ double vertical_angle = asin(spherical(1)); double horizontal_angle = atan2(spherical(0), spherical(2)); - double latitude = ((vertical_angle + M_PI_2) / M_PI) * height; - double longitude = ((horizontal_angle + M_PI) / (2.0 * M_PI)) * width; + double latitude = ((vertical_angle + M_PI_2) / M_PI) * height; + double longitude = ((horizontal_angle + M_PI) / (2.0 * M_PI)) * width; return Vec2(longitude, latitude); } - -} -} \ No newline at end of file + +} // namespace SphericalMapping +} // namespace aliceVision \ No newline at end of file diff --git a/src/aliceVision/panorama/sphericalMapping.hpp b/src/aliceVision/panorama/sphericalMapping.hpp index 968dd2baab..3bb4746da8 100644 --- a/src/aliceVision/panorama/sphericalMapping.hpp +++ b/src/aliceVision/panorama/sphericalMapping.hpp @@ -2,7 +2,7 @@ #include -namespace aliceVision +namespace aliceVision { namespace SphericalMapping @@ -14,7 +14,7 @@ namespace SphericalMapping * @param height number of pixels used to represent latitude * @return spherical coordinates */ -Vec3 fromEquirectangular(const Vec2 & equirectangular, int width, int height); +Vec3 fromEquirectangular(const Vec2& equirectangular, int width, int height); /** * Map from Spherical to equirectangular coordinates @@ -23,7 +23,7 @@ Vec3 fromEquirectangular(const Vec2 & equirectangular, int width, int height); * @param height number of pixels used to represent latitude * @return equirectangular coordinates */ -Vec2 toEquirectangular(const Vec3 & spherical, int width, int height); - -} -} \ No newline at end of file +Vec2 toEquirectangular(const Vec3& spherical, int width, int height); + +} // namespace SphericalMapping +} // namespace aliceVision \ No newline at end of file diff --git a/src/aliceVision/panorama/warper.cpp b/src/aliceVision/panorama/warper.cpp index 760b3bb9cb..b4a36ca91b 100644 --- a/src/aliceVision/panorama/warper.cpp +++ b/src/aliceVision/panorama/warper.cpp @@ -1,8 +1,10 @@ #include "warper.hpp" -namespace aliceVision { +namespace aliceVision +{ -bool Warper::warp(const CoordinatesMap & map, const aliceVision::image::Image & source) { +bool Warper::warp(const CoordinatesMap& map, const aliceVision::image::Image& source) +{ /** * Copy additional info from map @@ -12,36 +14,40 @@ bool Warper::warp(const CoordinatesMap & map, const aliceVision::image::Image sampler; - const aliceVision::image::Image & coordinates = map.getCoordinates(); + const aliceVision::image::Image& coordinates = map.getCoordinates(); /** * Create buffer * No longer need to keep a 2**x size */ _color = aliceVision::image::Image(coordinates.Width(), coordinates.Height()); - + /** * Simple warp */ - for (size_t i = 0; i < _color.Height(); i++) { - for (size_t j = 0; j < _color.Width(); j++) { + for(size_t i = 0; i < _color.Height(); i++) + { + for(size_t j = 0; j < _color.Width(); j++) + { - bool valid = _mask(i, j); - if (!valid) { - continue; - } + bool valid = _mask(i, j); + if(!valid) + { + continue; + } - const Eigen::Vector2d & coord = coordinates(i, j); - const image::RGBfColor pixel = sampler(source, coord(1), coord(0)); + const Eigen::Vector2d& coord = coordinates(i, j); + const image::RGBfColor pixel = sampler(source, coord(1), coord(0)); - _color(i, j) = pixel; - } + _color(i, j) = pixel; + } } return true; } -bool GaussianWarper::warp(const CoordinatesMap & map, const GaussianPyramidNoMask & pyramid) { +bool GaussianWarper::warp(const CoordinatesMap& map, const GaussianPyramidNoMask& pyramid) +{ /** * Copy additional info from map @@ -51,66 +57,72 @@ bool GaussianWarper::warp(const CoordinatesMap & map, const GaussianPyramidNoMa _mask = map.getMask(); const image::Sampler2d sampler; - const aliceVision::image::Image & coordinates = map.getCoordinates(); + const aliceVision::image::Image& coordinates = map.getCoordinates(); /** * Create a pyramid for input */ - const std::vector> & mlsource = pyramid.getPyramidColor(); - size_t max_level = pyramid.getScalesCount() - 1; + const std::vector>& mlsource = pyramid.getPyramidColor(); + size_t max_level = pyramid.getScalesCount() - 1; /** * Create buffer */ - _color = aliceVision::image::Image(coordinates.Width(), coordinates.Height(), true, image::RGBfColor(1.0, 0.0, 0.0)); + _color = aliceVision::image::Image(coordinates.Width(), coordinates.Height(), true, + image::RGBfColor(1.0, 0.0, 0.0)); /** * Multi level warp */ - for (size_t i = 0; i < _color.Height(); i++) { - for (size_t j = 0; j < _color.Width(); j++) { - - bool valid = _mask(i, j); - if (!valid) { - continue; - } - - if (i == _color.Height() - 1 || j == _color.Width() - 1 || !_mask(i + 1, j) || !_mask(i, j + 1)) { - const Eigen::Vector2d & coord = coordinates(i, j); - const image::RGBfColor pixel = sampler(mlsource[0], coord(1), coord(0)); - _color(i, j) = pixel; - continue; - } - - const Eigen::Vector2d & coord_mm = coordinates(i, j); - const Eigen::Vector2d & coord_mp = coordinates(i, j + 1); - const Eigen::Vector2d & coord_pm = coordinates(i + 1, j); - - double dxx = coord_pm(0) - coord_mm(0); - double dxy = coord_mp(0) - coord_mm(0); - double dyx = coord_pm(1) - coord_mm(1); - double dyy = coord_mp(1) - coord_mm(1); - double det = std::abs(dxx*dyy - dxy*dyx); - double scale = sqrt(det); - - double flevel = std::max(0.0, log2(scale)); - size_t blevel = std::min(max_level, size_t(floor(flevel))); - - double dscale, x, y; - dscale = 1.0 / pow(2.0, blevel); - x = coord_mm(0) * dscale; - y = coord_mm(1) * dscale; - /*Fallback to first level if outside*/ - if (x >= mlsource[blevel].Width() - 1 || y >= mlsource[blevel].Height() - 1) { - _color(i, j) = sampler(mlsource[0], coord_mm(1), coord_mm(0)); - continue; + for(size_t i = 0; i < _color.Height(); i++) + { + for(size_t j = 0; j < _color.Width(); j++) + { + + bool valid = _mask(i, j); + if(!valid) + { + continue; + } + + if(i == _color.Height() - 1 || j == _color.Width() - 1 || !_mask(i + 1, j) || !_mask(i, j + 1)) + { + const Eigen::Vector2d& coord = coordinates(i, j); + const image::RGBfColor pixel = sampler(mlsource[0], coord(1), coord(0)); + _color(i, j) = pixel; + continue; + } + + const Eigen::Vector2d& coord_mm = coordinates(i, j); + const Eigen::Vector2d& coord_mp = coordinates(i, j + 1); + const Eigen::Vector2d& coord_pm = coordinates(i + 1, j); + + double dxx = coord_pm(0) - coord_mm(0); + double dxy = coord_mp(0) - coord_mm(0); + double dyx = coord_pm(1) - coord_mm(1); + double dyy = coord_mp(1) - coord_mm(1); + double det = std::abs(dxx * dyy - dxy * dyx); + double scale = sqrt(det); + + double flevel = std::max(0.0, log2(scale)); + size_t blevel = std::min(max_level, size_t(floor(flevel))); + + double dscale, x, y; + dscale = 1.0 / pow(2.0, blevel); + x = coord_mm(0) * dscale; + y = coord_mm(1) * dscale; + /*Fallback to first level if outside*/ + if(x >= mlsource[blevel].Width() - 1 || y >= mlsource[blevel].Height() - 1) + { + _color(i, j) = sampler(mlsource[0], coord_mm(1), coord_mm(0)); + continue; + } + + _color(i, j) = sampler(mlsource[blevel], y, x); } - - _color(i, j) = sampler(mlsource[blevel], y, x); - } } return true; - } +} - } \ No newline at end of file +} // namespace aliceVision \ No newline at end of file diff --git a/src/aliceVision/panorama/warper.hpp b/src/aliceVision/panorama/warper.hpp index d7f15bca15..7039e90a4b 100644 --- a/src/aliceVision/panorama/warper.hpp +++ b/src/aliceVision/panorama/warper.hpp @@ -3,41 +3,34 @@ #include "coordinatesMap.hpp" #include "gaussian.hpp" +namespace aliceVision +{ -namespace aliceVision { - -class Warper { +class Warper +{ public: - virtual bool warp(const CoordinatesMap & map, const aliceVision::image::Image & source); + virtual bool warp(const CoordinatesMap& map, const aliceVision::image::Image& source); - const aliceVision::image::Image & getColor() const { - return _color; - } + const aliceVision::image::Image& getColor() const { return _color; } - const aliceVision::image::Image & getMask() const { - return _mask; - } - + const aliceVision::image::Image& getMask() const { return _mask; } - size_t getOffsetX() const { - return _offset_x; - } + size_t getOffsetX() const { return _offset_x; } - size_t getOffsetY() const { - return _offset_y; - } + size_t getOffsetY() const { return _offset_y; } protected: - size_t _offset_x = 0; - size_t _offset_y = 0; - - aliceVision::image::Image _color; - aliceVision::image::Image _mask; + size_t _offset_x = 0; + size_t _offset_y = 0; + + aliceVision::image::Image _color; + aliceVision::image::Image _mask; }; -class GaussianWarper : public Warper { +class GaussianWarper : public Warper +{ public: - virtual bool warp(const CoordinatesMap & map, const GaussianPyramidNoMask & pyramid); + virtual bool warp(const CoordinatesMap& map, const GaussianPyramidNoMask& pyramid); }; -} \ No newline at end of file +} // namespace aliceVision \ No newline at end of file diff --git a/src/software/pipeline/CMakeLists.txt b/src/software/pipeline/CMakeLists.txt index aae1c1b152..82e9995b6f 100644 --- a/src/software/pipeline/CMakeLists.txt +++ b/src/software/pipeline/CMakeLists.txt @@ -144,6 +144,7 @@ if(ALICEVISION_BUILD_SFM) aliceVision_image aliceVision_sfmData aliceVision_sfmDataIO + aliceVision_panorama ${Boost_LIBRARIES} ) alicevision_add_software(aliceVision_panoramaInit diff --git a/src/software/pipeline/main_panoramaCompositing.cpp b/src/software/pipeline/main_panoramaCompositing.cpp index a6363bf025..c61c5d5e9b 100644 --- a/src/software/pipeline/main_panoramaCompositing.cpp +++ b/src/software/pipeline/main_panoramaCompositing.cpp @@ -27,8 +27,18 @@ #include #include -#include -#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + // These constants define the current software version. // They must be updated when the command line is changed. @@ -41,2104 +51,6 @@ namespace po = boost::program_options; namespace bpt = boost::property_tree; namespace fs = boost::filesystem; -typedef struct { - size_t offset_x; - size_t offset_y; - std::string img_path; - std::string mask_path; - std::string weights_path; -} ConfigView; - - -/** - * @brief Maxflow computation based on a standard Adjacency List graph reprensentation. - * - * @see MaxFlow_CSR which use less memory. - */ -class MaxFlow_AdjList -{ -public: - using NodeType = int; - using ValueType = float; - - using Traits = boost::adjacency_list_traits< - boost::vecS, // OutEdgeListS - boost::vecS, // VertexListS - boost::directedS, - boost::vecS // EdgeListS - >; - using edge_descriptor = typename Traits::edge_descriptor; - using vertex_descriptor = typename Traits::vertex_descriptor; - using vertex_size_type = typename Traits::vertices_size_type; - struct Edge { - ValueType capacity{}; - ValueType residual{}; - edge_descriptor reverse; - }; - using Graph = boost::adjacency_list; - using VertexIterator = typename boost::graph_traits::vertex_iterator; - -public: - explicit MaxFlow_AdjList(size_t numNodes) - : _graph(numNodes+2) - , _S(NodeType(numNodes)) - , _T(NodeType(numNodes+1)) - { - VertexIterator vi, vi_end; - for(boost::tie(vi, vi_end) = vertices(_graph); vi != vi_end; ++vi) - { - _graph.m_vertices[*vi].m_out_edges.reserve(9); - } - _graph.m_vertices[numNodes].m_out_edges.reserve(numNodes); - _graph.m_vertices[numNodes+1].m_out_edges.reserve(numNodes); - } - - inline void addNodeToSource(NodeType n, ValueType source) - { - assert(source >= 0); - - edge_descriptor edge(boost::add_edge(_S, n, _graph).first); - edge_descriptor reverseEdge(boost::add_edge(n, _S, _graph).first); - - _graph[edge].capacity = source; - _graph[edge].reverse = reverseEdge; - _graph[reverseEdge].reverse = edge; - _graph[reverseEdge].capacity = source; - } - - inline void addNodeToSink(NodeType n, ValueType sink) - { - assert(sink >= 0); - - edge_descriptor edge(boost::add_edge(_T, n, _graph).first); - edge_descriptor reverseEdge(boost::add_edge(n, _T, _graph).first); - - _graph[edge].capacity = sink; - _graph[edge].reverse = reverseEdge; - _graph[reverseEdge].reverse = edge; - _graph[reverseEdge].capacity = sink; - } - - inline void addEdge(NodeType n1, NodeType n2, ValueType capacity, ValueType reverseCapacity) - { - assert(capacity >= 0 && reverseCapacity >= 0); - - edge_descriptor edge(boost::add_edge(n1, n2, _graph).first); - edge_descriptor reverseEdge(boost::add_edge(n2, n1, _graph).first); - _graph[edge].capacity = capacity; - _graph[edge].reverse = reverseEdge; - - _graph[reverseEdge].capacity = reverseCapacity; - _graph[reverseEdge].reverse = edge; - } - - void printStats() const; - void printColorStats() const; - - inline ValueType compute() - { - vertex_size_type nbVertices(boost::num_vertices(_graph)); - _color.resize(nbVertices, boost::white_color); - std::vector pred(nbVertices); - std::vector dist(nbVertices); - - ValueType v = boost::boykov_kolmogorov_max_flow(_graph, - boost::get(&Edge::capacity, _graph), - boost::get(&Edge::residual, _graph), - boost::get(&Edge::reverse, _graph), - &pred[0], - &_color[0], - &dist[0], - boost::get(boost::vertex_index, _graph), - _S, _T - ); - - return v; - } - - /// is empty - inline bool isSource(NodeType n) const - { - return (_color[n] == boost::black_color); - } - /// is full - inline bool isTarget(NodeType n) const - { - return (_color[n] == boost::white_color); - } - -protected: - Graph _graph; - std::vector _color; - const NodeType _S; //< emptyness - const NodeType _T; //< fullness -}; - -bool computeSeamsMap(image::Image & seams, const image::Image & labels) { - - if (seams.size() != labels.size()) { - return false; - } - - seams.fill(0); - - for (int j = 1; j < labels.Width() - 1; j++) { - IndexT label = labels(0, j); - IndexT same = true; - - same &= (labels(0, j - 1) == label); - same &= (labels(0, j + 1) == label); - same &= (labels(1, j - 1) == label); - same &= (labels(1, j) == label); - same &= (labels(1, j + 1) == label); - - if (same) { - continue; - } - - seams(0, j) = 255; - } - - int lastrow = labels.Height() - 1; - for (int j = 1; j < labels.Width() - 1; j++) { - IndexT label = labels(lastrow, j); - IndexT same = true; - - same &= (labels(lastrow - 1, j - 1) == label); - same &= (labels(lastrow - 1, j + 1) == label); - same &= (labels(lastrow, j - 1) == label); - same &= (labels(lastrow, j ) == label); - same &= (labels(lastrow, j + 1) == label); - - if (same) { - continue; - } - - seams(lastrow, j) = 255; - } - - for (int i = 1; i < labels.Height() - 1; i++) { - - for (int j = 1; j < labels.Width() - 1; j++) { - - IndexT label = labels(i, j); - IndexT same = true; - - same &= (labels(i - 1, j - 1) == label); - same &= (labels(i - 1, j) == label); - same &= (labels(i - 1, j + 1) == label); - same &= (labels(i, j - 1) == label); - same &= (labels(i, j + 1) == label); - same &= (labels(i + 1, j - 1) == label); - same &= (labels(i + 1, j) == label); - same &= (labels(i + 1, j + 1) == label); - - if (same) { - continue; - } - - seams(i, j) = 255; - } - } - - return true; -} - -static inline int f (int x_i, int gi) noexcept { - return (x_i*x_i)+gi*gi; -} - -static inline int sep (int i, int u, int gi, int gu, int) noexcept { - return (u*u - i*i + gu*gu - gi*gi) / (2*(u-i)); -} - -/// Code adapted from VFLib: https://github.com/vinniefalco/VFLib (Licence MIT) -bool computeDistanceMap(image::Image & distance, const image::Image & mask) { - - int width = mask.Width(); - int height = mask.Height(); - - int maxval = width + height; - image::Image buf(width, height); - - /* Per column distance 1D calculation */ - for (int j = 0; j < width; j++) - { - buf(0, j) = mask(0, j) ? 0 : maxval; - - /*Top to bottom accumulation */ - for (int i = 1; i < height; i++) { - - buf(i, j) = mask(i, j) ? 0 : 1 + buf(i - 1, j); - } - - /*Bottom to top correction */ - for (int i = height - 2; i >=0; i--) { - - if (buf(i + 1, j) < buf(i, j)) { - buf(i, j) = 1 + buf(i + 1, j); - } - } - } - - std::vector s (std::max(width, height)); - std::vector t (std::max(width, height)); - - /*Per row scan*/ - for (int i = 0; i < height; i++) - { - int q = 0; - s[0] = 0; - t[0] = 0; - - // scan 3 - for (int j = 1; j < width; j++) - { - while (q >= 0 && f(t[q]-s[q], buf(i, s[q])) > f(t[q]-j, buf(i, j))) - q--; - - if (q < 0) - { - q = 0; - s [0] = j; - } - else - { - int const w = 1 + sep (s[q], j, buf(i, s[q]), buf(i, j), maxval); - - if (w < width) - { - ++q; - s[q] = j; - t[q] = w; - } - } - } - - // scan 4 - for (int j = width - 1; j >= 0; --j) - { - int const d = f (j-s[q], buf(i, s[q])); - - distance(i, j) = d; - if (j == t[q]) - --q; - } - } - - - return true; -} - -bool feathering(aliceVision::image::Image & output, const aliceVision::image::Image & color, const aliceVision::image::Image & inputMask) { - - std::vector> feathering; - std::vector> feathering_mask; - feathering.push_back(color); - feathering_mask.push_back(inputMask); - - int lvl = 0; - int width = color.Width(); - int height = color.Height(); - - while (1) { - const image::Image & src = feathering[lvl]; - const image::Image & src_mask = feathering_mask[lvl]; - - image::Image half(width / 2, height / 2); - image::Image half_mask(width / 2, height / 2); - - for (int i = 0; i < half.Height(); i++) { - - int di = i * 2; - for (int j = 0; j < half.Width(); j++) { - int dj = j * 2; - - int count = 0; - half(i, j) = image::RGBfColor(0.0,0.0,0.0); - - if (src_mask(di, dj)) { - half(i, j) += src(di, dj); - count++; - } - - if (src_mask(di, dj + 1)) { - half(i, j) += src(di, dj + 1); - count++; - } - - if (src_mask(di + 1, dj)) { - half(i, j) += src(di + 1, dj); - count++; - } - - if (src_mask(di + 1, dj + 1)) { - half(i, j) += src(di + 1, dj + 1); - count++; - } - - if (count > 0) { - half(i, j) /= float(count); - half_mask(i, j) = 1; - } - else { - half_mask(i, j) = 0; - } - } - - - } - - feathering.push_back(half); - feathering_mask.push_back(half_mask); - - - width = half.Width(); - height = half.Height(); - - if (width < 2 || height < 2) break; - - lvl++; - } - - - for (int lvl = feathering.size() - 2; lvl >= 0; lvl--) { - - image::Image & src = feathering[lvl]; - image::Image & src_mask = feathering_mask[lvl]; - image::Image & ref = feathering[lvl + 1]; - image::Image & ref_mask = feathering_mask[lvl + 1]; - - for (int i = 0; i < src_mask.Height(); i++) { - for (int j = 0; j < src_mask.Width(); j++) { - if (!src_mask(i, j)) { - int mi = i / 2; - int mj = j / 2; - - if (mi >= ref_mask.Height()) { - mi = ref_mask.Height() - 1; - } - - if (mj >= ref_mask.Width()) { - mj = ref_mask.Width() - 1; - } - - src_mask(i, j) = ref_mask(mi, mj); - src(i, j) = ref(mi, mj); - } - } - } - } - - output = feathering[0]; - - return true; -} - -void drawBorders(aliceVision::image::Image & inout, aliceVision::image::Image & mask, size_t offset_x, size_t offset_y) { - - - for (int i = 0; i < mask.Height(); i++) { - int j = 0; - int di = i + offset_y; - int dj = j + offset_x; - if (dj >= inout.Width()) { - dj = dj - inout.Width(); - } - - if (mask(i, j)) { - inout(di, dj) = image::RGBAfColor(0.0f, 1.0f, 0.0f, 1.0f); - } - } - - for (int i = 0; i < mask.Height(); i++) { - int j = mask.Width() - 1; - int di = i + offset_y; - int dj = j + offset_x; - if (dj >= inout.Width()) { - dj = dj - inout.Width(); - } - - if (mask(i, j)) { - inout(di, dj) = image::RGBAfColor(0.0f, 1.0f, 0.0f, 1.0f); - } - } - - for (int j = 0; j < mask.Width(); j++) { - int i = 0; - int di = i + offset_y; - int dj = j + offset_x; - if (dj >= inout.Width()) { - dj = dj - inout.Width(); - } - - if (mask(i, j)) { - inout(di, dj) = image::RGBAfColor(0.0f, 1.0f, 0.0f, 1.0f); - } - } - - for (int j = 0; j < mask.Width(); j++) { - int i = mask.Height() - 1; - int di = i + offset_y; - int dj = j + offset_x; - if (dj >= inout.Width()) { - dj = dj - inout.Width(); - } - - if (mask(i, j)) { - inout(di, dj) = image::RGBAfColor(0.0f, 1.0f, 0.0f, 1.0f); - } - } - - for (int i = 1; i < mask.Height() - 1; i++) { - - int di = i + offset_y; - - for (int j = 1; j < mask.Width() - 1; j++) { - - int dj = j + offset_x; - if (dj >= inout.Width()) { - dj = dj - inout.Width(); - } - - if (!mask(i, j)) continue; - - unsigned char others = true; - others &= mask(i - 1, j - 1); - others &= mask(i - 1, j + 1); - others &= mask(i, j - 1); - others &= mask(i, j + 1); - others &= mask(i + 1, j - 1); - others &= mask(i + 1, j + 1); - if (others) continue; - - inout(di, dj) = image::RGBAfColor(0.0f, 1.0f, 0.0f, 1.0f); - } - } -} - -void drawSeams(aliceVision::image::Image & inout, aliceVision::image::Image & labels) { - - for (int i = 1; i < labels.Height() - 1; i++) { - - for (int j = 1; j < labels.Width() - 1; j++) { - - IndexT label = labels(i, j); - IndexT same = true; - - same &= (labels(i - 1, j - 1) == label); - same &= (labels(i - 1, j + 1) == label); - same &= (labels(i, j - 1) == label); - same &= (labels(i, j + 1) == label); - same &= (labels(i + 1, j - 1) == label); - same &= (labels(i + 1, j + 1) == label); - - if (same) { - continue; - } - - inout(i, j) = image::RGBAfColor(1.0f, 0.0f, 0.0f, 1.0f); - } - } -} - -void getMaskFromLabels(aliceVision::image::Image & mask, aliceVision::image::Image & labels, IndexT index, size_t offset_x, size_t offset_y) { - - for (int i = 0; i < mask.Height(); i++) { - - int di = i + offset_y; - - for (int j = 0; j < mask.Width(); j++) { - - int dj = j + offset_x; - if (dj >= labels.Width()) { - dj = dj - labels.Width(); - } - - - if (labels(di, dj) == index) { - mask(i, j) = 1.0f; - } - else { - mask(i, j) = 0.0f; - } - } - } -} - -template -bool makeImagePyramidCompatible(image::Image & output, size_t & out_offset_x, size_t & out_offset_y, const image::Image & input, size_t offset_x, size_t offset_y, size_t num_levels) { - - if (num_levels == 0) { - return false; - } - - double max_scale = 1.0 / pow(2.0, num_levels - 1); - - double low_offset_x = double(offset_x) * max_scale; - double low_offset_y = double(offset_y) * max_scale; - - /*Make sure offset is integer even at the lowest level*/ - double corrected_low_offset_x = floor(low_offset_x); - double corrected_low_offset_y = floor(low_offset_y); - - /*Add some borders on the top and left to make sure mask can be smoothed*/ - corrected_low_offset_x = std::max(0.0, corrected_low_offset_x - 3.0); - corrected_low_offset_y = std::max(0.0, corrected_low_offset_y - 3.0); - - /*Compute offset at largest level*/ - out_offset_x = size_t(corrected_low_offset_x / max_scale); - out_offset_y = size_t(corrected_low_offset_y / max_scale); - - /*Compute difference*/ - double doffset_x = double(offset_x) - double(out_offset_x); - double doffset_y = double(offset_y) - double(out_offset_y); - - /* update size with border update */ - double large_width = double(input.Width()) + doffset_x; - double large_height = double(input.Height()) + doffset_y; - - /* compute size at largest scale */ - double low_width = large_width * max_scale; - double low_height = large_height * max_scale; - - /*Make sure width is integer event at the lowest level*/ - double corrected_low_width = ceil(low_width); - double corrected_low_height = ceil(low_height); - - /*Add some borders on the right and bottom to make sure mask can be smoothed*/ - corrected_low_width = corrected_low_width + 3; - corrected_low_height = corrected_low_height + 3; - - /*Compute size at largest level*/ - size_t width = size_t(corrected_low_width / max_scale); - size_t height = size_t(corrected_low_height / max_scale); - - output = image::Image(width, height, true, T(0.0f)); - output.block(doffset_y, doffset_x, input.Height(), input.Width()) = input; - - return true; -} - - -class Compositer { -public: - Compositer(size_t outputWidth, size_t outputHeight) : - _panorama(outputWidth, outputHeight, true, image::RGBAfColor(0.0f, 0.0f, 0.0f, 0.0f)) - { - } - - virtual bool append(const aliceVision::image::Image & color, const aliceVision::image::Image & inputMask, const aliceVision::image::Image & inputWeights, size_t offset_x, size_t offset_y) { - - for (size_t i = 0; i < color.Height(); i++) { - - size_t pano_i = offset_y + i; - if (pano_i >= _panorama.Height()) { - continue; - } - - for (size_t j = 0; j < color.Width(); j++) { - - if (!inputMask(i, j)) { - continue; - } - - size_t pano_j = offset_x + j; - if (pano_j >= _panorama.Width()) { - pano_j = pano_j - _panorama.Width(); - } - - _panorama(pano_i, pano_j).r() = color(i, j).r(); - _panorama(pano_i, pano_j).g() = color(i, j).g(); - _panorama(pano_i, pano_j).b() = color(i, j).b(); - _panorama(pano_i, pano_j).a() = 1.0f; - } - } - - return true; - } - - virtual bool terminate() { - return true; - } - - aliceVision::image::Image & getPanorama() { - return _panorama; - } - -protected: - aliceVision::image::Image _panorama; -}; - -class AlphaCompositer : public Compositer { -public: - - AlphaCompositer(size_t outputWidth, size_t outputHeight) : - Compositer(outputWidth, outputHeight) { - - } - - virtual bool append(const aliceVision::image::Image & color, const aliceVision::image::Image & inputMask, const aliceVision::image::Image & inputWeights, size_t offset_x, size_t offset_y) { - - for (size_t i = 0; i < color.Height(); i++) { - - size_t pano_i = offset_y + i; - if (pano_i >= _panorama.Height()) { - continue; - } - - for (size_t j = 0; j < color.Width(); j++) { - - if (!inputMask(i, j)) { - continue; - } - - size_t pano_j = offset_x + j; - if (pano_j >= _panorama.Width()) { - pano_j = pano_j - _panorama.Width(); - } - - float wc = inputWeights(i, j); - - _panorama(pano_i, pano_j).r() += wc * color(i, j).r(); - _panorama(pano_i, pano_j).g() += wc * color(i, j).g(); - _panorama(pano_i, pano_j).b() += wc * color(i, j).b(); - _panorama(pano_i, pano_j).a() += wc; - } - } - - return true; - } - - virtual bool terminate() { - - for (int i = 0; i < _panorama.Height(); i++) { - for (int j = 0; j < _panorama.Width(); j++) { - - if (_panorama(i, j).a() < 1e-6) { - _panorama(i, j).r() = 1.0f; - _panorama(i, j).g() = 0.0f; - _panorama(i, j).b() = 0.0f; - _panorama(i, j).a() = 0.0f; - } - else { - _panorama(i, j).r() = _panorama(i, j).r() / _panorama(i, j).a(); - _panorama(i, j).g() = _panorama(i, j).g() / _panorama(i, j).a(); - _panorama(i, j).b() = _panorama(i, j).b() / _panorama(i, j).a(); - _panorama(i, j).a() = 1.0f; - } - } - } - - return true; - } -}; - -template -inline void convolveRow(typename image::Image::RowXpr output_row, typename image::Image::ConstRowXpr input_row, const Eigen::Matrix & kernel, bool loop) { - - const int radius = 2; - - for (int j = 0; j < input_row.cols(); j++) { - - T sum = T(); - float sumw = 0.0f; - - for (int k = 0; k < kernel.size(); k++) { - - float w = kernel(k); - int col = j + k - radius; - - /* mirror 5432 | 123456 | 5432 */ - - if (!loop) { - if (col < 0) { - col = - col; - } - - if (col >= input_row.cols()) { - col = input_row.cols() - 1 - (col + 1 - input_row.cols()); - } - } - else { - if (col < 0) { - col = input_row.cols() + col; - } - - if (col >= input_row.cols()) { - col = col - input_row.cols(); - } - } - - sum += w * input_row(col); - sumw += w; - } - - output_row(j) = sum / sumw; - } -} - -template -inline void convolveColumns(typename image::Image::RowXpr output_row, const image::Image & input_rows, const Eigen::Matrix & kernel) { - - for (int j = 0; j < output_row.cols(); j++) { - - T sum = T(); - float sumw = 0.0f; - - for (int k = 0; k < kernel.size(); k++) { - - float w = kernel(k); - sum += w * input_rows(k, j); - sumw += w; - } - - output_row(j) = sum / sumw; - } -} - -template -bool convolveGaussian5x5(image::Image & output, const image::Image & input, bool loop = false) { - - if (output.size() != input.size()) { - return false; - } - - Eigen::Matrix kernel; - kernel[0] = 1.0f; - kernel[1] = 4.0f; - kernel[2] = 6.0f; - kernel[3] = 4.0f; - kernel[4] = 1.0f; - kernel = kernel / kernel.sum(); - - image::Image buf(output.Width(), 5); - - int radius = 2; - - convolveRow(buf.row(0), input.row(2), kernel, loop); - convolveRow(buf.row(1), input.row(1), kernel, loop); - convolveRow(buf.row(2), input.row(0), kernel, loop); - convolveRow(buf.row(3), input.row(1), kernel, loop); - convolveRow(buf.row(4), input.row(2), kernel, loop); - - for (int i = 0; i < output.Height() - 3; i++) { - - convolveColumns(output.row(i), buf, kernel); - - - buf.row(0) = buf.row(1); - buf.row(1) = buf.row(2); - buf.row(2) = buf.row(3); - buf.row(3) = buf.row(4); - convolveRow(buf.row(4), input.row(i + 3), kernel, loop); - } - - /** - current row : -5 -4 -3 -2 -1 - next 1 : -4 -3 -2 -1 -2 - next 2 : -3 -2 -1 -2 -3 - */ - convolveColumns(output.row(output.Height() - 3), buf, kernel); - - buf.row(0) = buf.row(1); - buf.row(1) = buf.row(2); - buf.row(2) = buf.row(3); - buf.row(3) = buf.row(4); - convolveRow(buf.row(4), input.row(output.Height() - 2), kernel, loop); - convolveColumns(output.row(output.Height() - 2), buf, kernel); - - buf.row(0) = buf.row(1); - buf.row(1) = buf.row(2); - buf.row(2) = buf.row(3); - buf.row(3) = buf.row(4); - convolveRow(buf.row(4), input.row(output.Height() - 3), kernel, loop); - convolveColumns(output.row(output.Height() - 1), buf, kernel); - - return true; -} - - -template -bool downscale(aliceVision::image::Image & outputColor, const aliceVision::image::Image & inputColor) { - - size_t output_width = inputColor.Width() / 2; - size_t output_height = inputColor.Height() / 2; - - for (int i = 0; i < output_height; i++) { - for (int j = 0; j < output_width; j++) { - outputColor(i, j) = inputColor(i * 2, j * 2); - } - } - - return true; -} - -template -bool upscale(aliceVision::image::Image & outputColor, const aliceVision::image::Image & inputColor) { - - size_t width = inputColor.Width(); - size_t height = inputColor.Height(); - - for (int i = 0; i < height; i++) { - - int di = i * 2; - - for (int j = 0; j < width; j++) { - int dj = j * 2; - - outputColor(di, dj) = T(); - outputColor(di, dj + 1) = T(); - outputColor(di + 1, dj) = T(); - outputColor(di + 1, dj + 1) = inputColor(i, j); - } - } - - return true; -} - -template -bool substract(aliceVision::image::Image & AminusB, const aliceVision::image::Image & A, const aliceVision::image::Image & B) { - - size_t width = AminusB.Width(); - size_t height = AminusB.Height(); - - if (AminusB.size() != A.size()) { - return false; - } - - if (AminusB.size() != B.size()) { - return false; - } - - for (int i = 0; i < height; i++) { - - for (int j = 0; j < width; j++) { - - AminusB(i, j) = A(i, j) - B(i, j); - } - } - - return true; -} - -template -bool addition(aliceVision::image::Image & AplusB, const aliceVision::image::Image & A, const aliceVision::image::Image & B) { - - size_t width = AplusB.Width(); - size_t height = AplusB.Height(); - - if (AplusB.size() != A.size()) { - return false; - } - - if (AplusB.size() != B.size()) { - return false; - } - - for (int i = 0; i < height; i++) { - - for (int j = 0; j < width; j++) { - - AplusB(i, j) = A(i, j) + B(i, j); - } - } - - return true; -} - -void removeNegativeValues(aliceVision::image::Image & img) { - for (int i = 0; i < img.Height(); i++) { - for (int j = 0; j < img.Width(); j++) { - image::RGBfColor & pix = img(i, j); - image::RGBfColor rpix; - rpix.r() = std::exp(pix.r()); - rpix.g() = std::exp(pix.g()); - rpix.b() = std::exp(pix.b()); - - if (rpix.r() < 0.0) { - pix.r() = 0.0; - } - - if (rpix.g() < 0.0) { - pix.g() = 0.0; - } - - if (rpix.b() < 0.0) { - pix.b() = 0.0; - } - } - } -} - -class LaplacianPyramid { -public: - LaplacianPyramid(size_t base_width, size_t base_height, size_t max_levels) { - - size_t width = base_width; - size_t height = base_height; - - /*Make sure pyramid size can be divided by 2 on each levels*/ - double max_scale = 1.0 / pow(2.0, max_levels - 1); - //width = size_t(ceil(double(width) * max_scale) / max_scale); - //height = size_t(ceil(double(height) * max_scale) / max_scale); - - /*Prepare pyramid*/ - for (int lvl = 0; lvl < max_levels; lvl++) { - - _levels.push_back(aliceVision::image::Image(width, height, true, image::RGBfColor(0.0f,0.0f,0.0f))); - _weights.push_back(aliceVision::image::Image(width, height, true, 0.0f)); - - height /= 2; - width /= 2; - } - } - - bool augment(size_t new_max_levels) { - - if (new_max_levels <= _levels.size()) { - return false; - } - - size_t old_max_level = _levels.size(); - - image::Image current_color = _levels[_levels.size() - 1]; - image::Image current_weights = _weights[_weights.size() - 1]; - - _levels[_levels.size() - 1].fill(image::RGBfColor(0.0f, 0.0f, 0.0f)); - _weights[_weights.size() - 1].fill(0.0f); - - image::Image current_mask(current_color.Width(), current_color.Height(), true, 0); - image::Image current_color_feathered(current_color.Width(), current_color.Height()); - - for (int i = 0; i < current_color.Height(); i++) { - for (int j = 0; j < current_color.Width(); j++) { - if (current_weights(i, j) < 1e-6) { - current_color(i, j) = image::RGBfColor(0.0); - continue; - } - - current_color(i, j).r() = current_color(i, j).r() / current_weights(i, j); - current_color(i, j).g() = current_color(i, j).g() / current_weights(i, j); - current_color(i, j).b() = current_color(i, j).b() / current_weights(i, j); - current_mask(i ,j) = 255; - } - } - - - feathering(current_color_feathered, current_color, current_mask); - current_color = current_color_feathered; - - - for (int l = old_max_level; l < new_max_levels; l++) { - - _levels.emplace_back(_levels[l - 1].Width() / 2, _levels[l - 1].Height() / 2, true, image::RGBfColor(0.0f, 0.0f, 0.0f)); - _weights.emplace_back(_weights[l - 1].Width() / 2, _weights[l - 1].Height() / 2, true, 0.0f); - } - - int width = current_color.Width(); - int height = current_color.Height(); - image::Image next_color; - image::Image next_weights; - - for (int l = old_max_level - 1; l < new_max_levels - 1; l++) - { - aliceVision::image::Image buf(width, height); - aliceVision::image::Image buf2(width, height); - aliceVision::image::Image bufw(width, height); - - next_color = aliceVision::image::Image(width / 2, height / 2); - next_weights = aliceVision::image::Image(width / 2, height / 2); - - convolveGaussian5x5(buf, current_color); - downscale(next_color, buf); - - convolveGaussian5x5(bufw, current_weights); - downscale(next_weights, bufw); - - upscale(buf, next_color); - convolveGaussian5x5(buf2, buf); - - for (int i = 0; i < buf2.Height(); i++) { - for (int j = 0; j < buf2.Width(); j++) { - buf2(i,j) *= 4.0f; - } - } - - substract(current_color, current_color, buf2); - - merge(current_color, current_weights, l, 0, 0); - - current_color = next_color; - current_weights = next_weights; - width /= 2; - height /= 2; - } - - merge(current_color, current_weights, _levels.size() - 1, 0, 0); - - return true; - } - - bool apply(const aliceVision::image::Image & source, const aliceVision::image::Image & mask, const aliceVision::image::Image & weights, size_t offset_x, size_t offset_y) { - - int width = source.Width(); - int height = source.Height(); - - /* Convert mask to alpha layer */ - image::Image mask_float(width, height); - for (int i = 0; i < height; i++) { - for (int j = 0; j < width; j++) { - if (mask(i, j)) { - mask_float(i, j) = 1.0f; - } - else { - mask_float(i, j) = 0.0f; - } - } - } - - image::Image current_color = source; - image::Image next_color; - image::Image current_weights = weights; - image::Image next_weights; - image::Image current_mask = mask_float; - image::Image next_mask; - - for (int l = 0; l < _levels.size() - 1; l++) - { - aliceVision::image::Image buf_masked(width, height); - aliceVision::image::Image buf(width, height); - aliceVision::image::Image buf2(width, height); - aliceVision::image::Image buf_float(width, height); - - next_color = aliceVision::image::Image(width / 2, height / 2); - next_weights = aliceVision::image::Image(width / 2, height / 2); - next_mask = aliceVision::image::Image(width / 2, height / 2); - - /*Apply mask to content before convolution*/ - for (int i = 0; i < current_color.Height(); i++) { - for (int j = 0; j < current_color.Width(); j++) { - if (std::abs(current_mask(i, j)) > 1e-6) { - buf_masked(i, j) = current_color(i, j); - } - else { - buf_masked(i, j).r() = 0.0f; - buf_masked(i, j).g() = 0.0f; - buf_masked(i, j).b() = 0.0f; - current_weights(i, j) = 0.0f; - } - } - } - - convolveGaussian5x5(buf, buf_masked); - convolveGaussian5x5(buf_float, current_mask); - - /* - Normalize given mask - */ - for (int i = 0; i < current_color.Height(); i++) { - for (int j = 0; j < current_color.Width(); j++) { - - float m = buf_float(i, j); - - if (std::abs(m) > 1e-6) { - buf(i, j).r() = buf(i, j).r() / m; - buf(i, j).g() = buf(i, j).g() / m; - buf(i, j).b() = buf(i, j).b() / m; - buf_float(i, j) = 1.0f; - } - else { - buf(i, j).r() = 0.0f; - buf(i, j).g() = 0.0f; - buf(i, j).b() = 0.0f; - buf_float(i, j) = 0.0f; - } - } - } - - downscale(next_color, buf); - downscale(next_mask, buf_float); - - upscale(buf, next_color); - convolveGaussian5x5(buf2, buf); - - for (int i = 0; i < buf2.Height(); i++) { - for (int j = 0; j < buf2.Width(); j++) { - buf2(i,j) *= 4.0f; - } - } - - substract(current_color, current_color, buf2); - - convolveGaussian5x5(buf_float, current_weights); - downscale(next_weights, buf_float); - - merge(current_color, current_weights, l, offset_x, offset_y); - - - current_color = next_color; - current_weights = next_weights; - current_mask = next_mask; - - width /= 2; - height /= 2; - offset_x /= 2; - offset_y /= 2; - } - - merge(current_color, current_weights, _levels.size() - 1, offset_x, offset_y); - - return true; - } - - bool merge(const aliceVision::image::Image & oimg, const aliceVision::image::Image & oweight, size_t level, size_t offset_x, size_t offset_y) { - - image::Image & img = _levels[level]; - image::Image & weight = _weights[level]; - - for (int i = 0; i < oimg.Height(); i++) { - - int di = i + offset_y; - if (di >= img.Height()) continue; - - for (int j = 0; j < oimg.Width(); j++) { - - int dj = j + offset_x; - if (dj >= weight.Width()) { - dj = dj - weight.Width(); - } - - img(di, dj).r() += oimg(i, j).r() * oweight(i, j); - img(di, dj).g() += oimg(i, j).g() * oweight(i, j); - img(di, dj).b() += oimg(i, j).b() * oweight(i, j); - weight(di, dj) += oweight(i, j); - } - } - - return true; - } - - bool rebuild(image::Image & output) { - - for (int l = 0; l < _levels.size(); l++) { - for (int i = 0; i < _levels[l].Height(); i++) { - for (int j = 0; j < _levels[l].Width(); j++) { - if (_weights[l](i, j) < 1e-6) { - _levels[l](i, j) = image::RGBfColor(0.0); - continue; - } - - _levels[l](i, j).r() = _levels[l](i, j).r() / _weights[l](i, j); - _levels[l](i, j).g() = _levels[l](i, j).g() / _weights[l](i, j); - _levels[l](i, j).b() = _levels[l](i, j).b() / _weights[l](i, j); - } - } - } - - removeNegativeValues(_levels[_levels.size() - 1]); - - for (int l = _levels.size() - 2; l >= 0; l--) { - - aliceVision::image::Image buf(_levels[l].Width(), _levels[l].Height()); - aliceVision::image::Image buf2(_levels[l].Width(), _levels[l].Height()); - - upscale(buf, _levels[l + 1]); - convolveGaussian5x5(buf2, buf, true); - - for (int i = 0; i < buf2.Height(); i++) { - for (int j = 0; j < buf2.Width(); j++) { - buf2(i,j) *= 4.0f; - } - } - - addition(_levels[l], _levels[l], buf2); - removeNegativeValues(_levels[l]); - } - - // Write output to RGBA - for (int i = 0; i < output.Height(); i++) { - for (int j = 0; j < output.Width(); j++) { - output(i, j).r() = _levels[0](i, j).r(); - output(i, j).g() = _levels[0](i, j).g(); - output(i, j).b() = _levels[0](i, j).b(); - - if (_weights[0](i, j) < 1e-6) { - output(i, j).a() = 0.0f; - } - else { - output(i, j).a() = 1.0f; - } - } - } - - return true; - } - -private: - std::vector> _levels; - std::vector> _weights; -}; - -class DistanceSeams { -public: - DistanceSeams(size_t outputWidth, size_t outputHeight) : - _weights(outputWidth, outputHeight, true, 0.0f), - _labels(outputWidth, outputHeight, true, 255) - { - } - virtual ~DistanceSeams() = default; - - virtual bool append(const aliceVision::image::Image & inputMask, const aliceVision::image::Image & inputWeights, IndexT currentIndex, size_t offset_x, size_t offset_y) - { - if (inputMask.size() != inputWeights.size()) { - return false; - } - - for (int i = 0; i < inputMask.Height(); i++) { - - int di = i + offset_y; - - for (int j = 0; j < inputMask.Width(); j++) { - - if (!inputMask(i, j)) { - continue; - } - - int dj = j + offset_x; - if (dj >= _weights.Width()) { - dj = dj - _weights.Width(); - } - - if (inputWeights(i, j) > _weights(di, dj)) { - _labels(di, dj) = currentIndex; - _weights(di, dj) = inputWeights(i, j); - } - } - } - - return true; - } - - const image::Image & getLabels() { - return _labels; - } - -private: - image::Image _weights; - image::Image _labels; -}; - -class GraphcutSeams { -public: - struct Rect { - int l; - int t; - int w; - int h; - }; - - using PixelInfo = std::pair; - using ImageOwners = image::Image>; - -public: - GraphcutSeams(size_t outputWidth, size_t outputHeight) : - _owners(outputWidth, outputHeight, true), - _labels(outputWidth, outputHeight, true, 0), - _original_labels(outputWidth, outputHeight, true, 0), - _distancesSeams(outputWidth, outputHeight, true, 0), - _maximal_distance_change(outputWidth + outputHeight) - { - } - - virtual ~GraphcutSeams() = default; - - void setOriginalLabels(const image::Image & existing_labels) { - - _labels = existing_labels; - _original_labels = existing_labels; - - image::Image seams(_labels.Width(), _labels.Height()); - computeSeamsMap(seams, _labels); - computeDistanceMap(_distancesSeams, seams); - } - - virtual bool append(const aliceVision::image::Image & input, const aliceVision::image::Image & inputMask, IndexT currentIndex, size_t offset_x, size_t offset_y) - { - - if (inputMask.size() != input.size()) { - return false; - } - - Rect rect; - - rect.l = offset_x; - rect.t = offset_y; - rect.w = input.Width() + 1; - rect.h = input.Height() + 1; - - /*Extend rect for borders*/ - rect.l = std::max(0, rect.l - 3); - rect.t = std::max(0, rect.t - 3); - rect.w = rect.w + 6; - rect.h = rect.h + 6; - if (rect.t + rect.h > _owners.Height()) { - rect.h = _owners.Height() - rect.t; - } - - _rects[currentIndex] = rect; - - - /* - _owners will get for each pixel of the panorama a list of pixels - in the sources which may have seen this point. - */ - for (int i = 0; i < input.Height(); i++) { - - int di = i + offset_y; - - for (int j = 0; j < input.Width(); j++) { - - if (!inputMask(i, j)) continue; - - int dj = j + offset_x; - if (dj >= _owners.Width()) { - dj = dj - _owners.Width(); - } - - PixelInfo info; - info.first = currentIndex; - info.second = input(i, j); - - /* If too far away from seam, do not add a contender */ - int dist = _distancesSeams(di, dj); - if (dist > _maximal_distance_change + 10) { - continue; - } - - _owners(di, dj).push_back(info); - } - } - - return true; - } - - void setMaximalDistance(int dist) { - _maximal_distance_change = dist; - } - - bool process() { - - - for (int i = 0; i < 10; i++) { - - /*For each possible label, try to extends its domination on the label's world */ - bool change = false; - - for (auto & info : _rects) { - - ALICEVISION_LOG_INFO("Graphcut expansion (iteration " << i << ") for label " << info.first); - - - int p1 = info.second.l; - int w1 = info.second.w; - int p2 = 0; - int w2 = 0; - - if (p1 + w1 > _labels.Width()) { - w1 = _labels.Width() - p1; - p2 = 0; - w2 = info.second.w - w1; - } - - Eigen::Matrix backup_1 = _labels.block(info.second.t, p1, info.second.h, w1); - Eigen::Matrix backup_2 = _labels.block(info.second.t, p2, info.second.h, w2); - - double base_cost = cost(info.first); - alphaExpansion(info.first); - double new_cost = cost(info.first); - - if (new_cost > base_cost) { - _labels.block(info.second.t, p1, info.second.h, w1) = backup_1; - _labels.block(info.second.t, p2, info.second.h, w2) = backup_2; - } - else if (new_cost < base_cost) { - change = true; - } - } - - if (!change) { - break; - } - } - - return true; - } - - double cost(IndexT currentLabel) { - - Rect rect = _rects[currentLabel]; - - double cost = 0.0; - - for (int i = 0; i < rect.h - 1; i++) { - - int y = rect.t + i; - int yp = y + 1; - - for (int j = 0; j < rect.w; j++) { - - int x = rect.l + j; - if (x >= _owners.Width()) { - x = x - _owners.Width(); - } - - int xp = x + 1; - if (xp >= _owners.Width()) { - xp = xp - _owners.Width(); - } - - IndexT label = _labels(y, x); - IndexT labelx = _labels(y, xp); - IndexT labely = _labels(yp, x); - - if (label == UndefinedIndexT) continue; - if (labelx == UndefinedIndexT) continue; - if (labely == UndefinedIndexT) continue; - - if (label == labelx) { - continue; - } - - image::RGBfColor CColorLC; - image::RGBfColor CColorLX; - image::RGBfColor CColorLY; - bool hasCLC = false; - bool hasCLX = false; - bool hasCLY = false; - - for (int l = 0; l < _owners(y, x).size(); l++) { - if (_owners(y, x)[l].first == label) { - hasCLC = true; - CColorLC = _owners(y, x)[l].second; - } - - if (_owners(y, x)[l].first == labelx) { - hasCLX = true; - CColorLX = _owners(y, x)[l].second; - } - - if (_owners(y, x)[l].first == labely) { - hasCLY = true; - CColorLY = _owners(y, x)[l].second; - } - } - - image::RGBfColor XColorLC; - image::RGBfColor XColorLX; - bool hasXLC = false; - bool hasXLX = false; - - for (int l = 0; l < _owners(y, xp).size(); l++) { - if (_owners(y, xp)[l].first == label) { - hasXLC = true; - XColorLC = _owners(y, xp)[l].second; - } - - if (_owners(y, xp)[l].first == labelx) { - hasXLX = true; - XColorLX = _owners(y, xp)[l].second; - } - } - - image::RGBfColor YColorLC; - image::RGBfColor YColorLY; - bool hasYLC = false; - bool hasYLY = false; - - for (int l = 0; l < _owners(yp, x).size(); l++) { - if (_owners(yp, x)[l].first == label) { - hasYLC = true; - YColorLC = _owners(yp, x)[l].second; - } - - if (_owners(yp, x)[l].first == labely) { - hasYLY = true; - YColorLY = _owners(yp, x)[l].second; - } - } - - if (!hasCLC || !hasXLX || !hasYLY) { - continue; - } - - - - if (!hasCLX) { - CColorLX = CColorLC; - } - - if (!hasCLY) { - CColorLY = CColorLC; - } - - if (!hasXLC) { - XColorLC = XColorLX; - } - - if (!hasYLC) { - YColorLC = YColorLY; - } - - cost += (CColorLC - CColorLX).norm(); - cost += (CColorLC - CColorLY).norm(); - cost += (XColorLC - XColorLX).norm(); - cost += (YColorLC - YColorLY).norm(); - } - } - - return cost; - } - - bool alphaExpansion(IndexT currentLabel) { - - Rect rect = _rects[currentLabel]; - - image::Image mask(rect.w, rect.h, true, 0); - image::Image ids(rect.w, rect.h, true, -1); - image::Image color_label(rect.w, rect.h, true, image::RGBfColor(0.0f, 0.0f, 0.0f)); - image::Image color_other(rect.w, rect.h, true, image::RGBfColor(0.0f, 0.0f, 0.0f)); - - /*Compute distance map to seams*/ - image::Image distanceMap(rect.w, rect.h); - { - image::Image binarizedWorld(rect.w, rect.h); - - for (int i = 0; i < rect.h; i++) { - int y = rect.t + i; - - for (int j = 0; j < rect.w; j++) { - - int x = rect.l + j; - if (x >= _owners.Width()) { - x = x - _owners.Width(); - } - - IndexT label = _original_labels(y, x); - if (label == currentLabel) { - binarizedWorld(i, j) = 1; - } - else { - binarizedWorld(i, j) = 0; - } - } - } - - image::Image seams(rect.w, rect.h); - if (!computeSeamsMap(seams, binarizedWorld)) { - return false; - } - - if (!computeDistanceMap(distanceMap, seams)) { - return false; - } - } - - - /* - A warped input has valid pixels only in some parts of the final image. - Rect is the bounding box of these valid pixels. - Let's build a mask : - - 0 if the pixel is not viewed by anyone - - 1 if the pixel is viewed by the current label alpha - - 2 if the pixel is viewed by *another* label and this label is marked as current valid label - - 3 if the pixel is 1 + 2 : the pixel is not selected as alpha territory, but alpha is looking at it - */ - for (int i = 0; i < rect.h; i++) { - - int y = rect.t + i; - - for (int j = 0; j < rect.w; j++) { - - int x = rect.l + j; - if (x >= _owners.Width()) { - x = x - _owners.Width(); - } - - std::vector & infos = _owners(y, x); - IndexT label = _labels(y, x); - - image::RGBfColor currentColor; - image::RGBfColor otherColor; - - int dist = distanceMap(i, j); - - /* Loop over observations */ - for (int l = 0; l < infos.size(); l++) { - - if (dist > _maximal_distance_change) { - - if (infos[l].first == label) { - if (label == currentLabel) { - mask(i, j) = 1; - currentColor = infos[l].second; - } - else { - mask(i, j) = 2; - otherColor = infos[l].second; - } - } - } - else { - if (infos[l].first == currentLabel) { - mask(i, j) |= 1; - currentColor = infos[l].second; - } - else if (infos[l].first == label) { - mask(i, j) |= 2; - otherColor = infos[l].second; - } - } - } - - /* - If the pixel may be a new kingdom for alpha ! - */ - if (mask(i, j) == 1) { - color_label(i, j) = currentColor; - color_other(i, j) = currentColor; - } - else if (mask(i, j) == 2) { - color_label(i, j) = otherColor; - color_other(i, j) = otherColor; - } - else if (mask(i, j) == 3) { - color_label(i, j) = currentColor; - color_other(i, j) = otherColor; - } - } - } - - /* - The rectangle is a grid. - However we want to ignore a lot of pixel. - Let's create an index per valid pixels for graph cut reference - */ - int count = 0; - for (int i = 0; i < rect.h; i++) { - for (int j = 0; j < rect.w; j++) { - if (mask(i, j) == 0) { - continue; - } - - ids(i, j) = count; - count++; - } - } - - /*Create graph*/ - MaxFlow_AdjList gc(count); - size_t countValid = 0; - - for (int i = 0; i < rect.h; i++) { - for (int j = 0; j < rect.w; j++) { - - /* If this pixel is not valid, ignore */ - if (mask(i, j) == 0) { - continue; - } - - /* Get this pixel ID */ - int node_id = ids(i, j); - - int im1 = std::max(i - 1, 0); - int jm1 = std::max(j - 1, 0); - int ip1 = std::min(i + 1, rect.h - 1); - int jp1 = std::min(j + 1, rect.w - 1); - - if (mask(i, j) == 1) { - - /* Only add nodes close to borders */ - if (mask(im1, jm1) == 1 && mask(im1, j) == 1 && mask(im1, jp1) == 1 && - mask(i, jm1) == 1 && mask(i, jp1) == 1 && - mask(ip1, jm1) == 1 && mask(ip1, j) == 1 && mask(ip1, jp1) == 1) { - continue; - } - - /* - This pixel is only seen by alpha. - Enforce its domination by stating that removing this pixel - from alpha territoy is infinitly costly (impossible). - */ - gc.addNodeToSource(node_id, 100000); - } - else if (mask(i, j) == 2) { - /* Only add nodes close to borders */ - if (mask(im1, jm1) == 2 && mask(im1, j) == 2 && mask(im1, jp1) == 2 && - mask(i, jm1) == 2 && mask(i, jp1) == 2 && - mask(ip1, jm1) == 2 && mask(ip1, j) == 2 && mask(ip1, jp1) == 2) { - continue; - } - - /* - This pixel is only seen by an ennemy. - Enforce its domination by stating that removing this pixel - from ennemy territory is infinitly costly (impossible). - */ - gc.addNodeToSink(node_id, 100000); - } - else if (mask(i, j) == 3) { - - /* - This pixel is seen by both alpha and enemies but is owned by ennemy. - Make sure that changing node owner will have no direct cost. - Connect it to both alpha and ennemy for the moment - (Graph cut will not allow a pixel to have both owners at the end). - */ - gc.addNodeToSource(node_id, 0); - gc.addNodeToSink(node_id, 0); - countValid++; - } - } - } - - if (countValid == 0) { - /* We have no possibility for territory expansion */ - /* let's exit */ - return true; - } - - /* - Loop over alpha bounding box. - Let's define the transition cost. - When two neighboor pixels have different labels, there is a seam (border) cost. - Graph cut will try to make sure the territory will have a minimal border cost - */ - for (int i = 0; i < rect.h; i++) { - for (int j = 0; j < rect.w; j++) { - - if (mask(i, j) == 0) { - continue; - } - - int node_id = ids(i, j); - - /* Make sure it is possible to estimate this horizontal border */ - if (i < mask.Height() - 1) { - - /* Make sure the other pixel is owned by someone */ - if (mask(i + 1, j)) { - - int other_node_id = ids(i + 1, j); - float w = 1000; - - - if (((mask(i, j) & 1) && (mask(i + 1, j) & 2)) || ((mask(i, j) & 2) && (mask(i + 1, j) & 1))) { - float d1 = (color_label(i, j) - color_other(i, j)).norm(); - float d2 = (color_label(i + 1, j) - color_other(i + 1, j)).norm(); - - d1 = std::min(2.0f, d1); - d2 = std::min(2.0f, d2); - - w = (d1 + d2) * 100.0 + 1.0; - } - - gc.addEdge(node_id, other_node_id, w, w); - } - } - - if (j < mask.Width() - 1) { - - if (mask(i, j + 1)) { - - int other_node_id = ids(i, j + 1); - float w = 1000; - - if (((mask(i, j) & 1) && (mask(i, j + 1) & 2)) || ((mask(i, j) & 2) && (mask(i, j + 1) & 1))) { - float d1 = (color_label(i, j) - color_other(i, j)).norm(); - float d2 = (color_label(i, j + 1) - color_other(i, j + 1)).norm(); - w = (d1 + d2) * 100.0 + 1.0; - } - - gc.addEdge(node_id, other_node_id, w, w); - } - } - } - } - - - gc.compute(); - - int changeCount = 0; - for (int i = 0; i < rect.h; i++) { - - int y = rect.t + i; - - for (int j = 0; j < rect.w; j++) { - - int x = rect.l + j; - if (x >= _owners.Width()) { - x = x - _owners.Width(); - } - - IndexT label = _labels(y, x); - int id = ids(i, j); - - if (gc.isSource(id)) { - - if (label != currentLabel) { - changeCount++; - } - - _labels(y, x) = currentLabel; - } - } - } - - return true; - } - - const image::Image & getLabels() { - - return _labels; - } - -private: - - std::map _rects; - ImageOwners _owners; - image::Image _labels; - image::Image _original_labels; - image::Image _distancesSeams; - size_t _maximal_distance_change; -}; - -class HierarchicalGraphcutSeams { -public: - - HierarchicalGraphcutSeams(size_t outputWidth, size_t outputHeight, size_t levelOfInterest): - _outputWidth(outputWidth), - _outputHeight(outputHeight), - _levelOfInterest(levelOfInterest), - _labels(outputWidth, outputHeight, true, UndefinedIndexT) { - - - double scale = 1.0 / pow(2.0, levelOfInterest); - size_t width = size_t(floor(double(outputWidth) * scale)); - size_t height = size_t(floor(double(outputHeight) * scale)); - - _graphcut = std::unique_ptr(new GraphcutSeams(width, height)); - } - - virtual ~HierarchicalGraphcutSeams() = default; - - void setOriginalLabels(const image::Image & labels) { - - /* - First of all, Propagate label to all levels - */ - image::Image current_label = labels; - - for (int l = 1; l <= _levelOfInterest; l++) { - - aliceVision::image::Image next_label(current_label.Width() / 2, current_label.Height() / 2); - - for (int i = 0; i < next_label.Height(); i++) { - int di = i * 2; - - for (int j = 0; j < next_label.Width(); j++) { - int dj = j * 2; - - next_label(i, j) = current_label(di, dj); - } - } - - current_label = next_label; - } - - _graphcut->setOriginalLabels(current_label); - } - - void setMaximalDistance(int distance) { - _graphcut->setMaximalDistance(distance); - } - - virtual bool append(const aliceVision::image::Image & input, const aliceVision::image::Image & inputMask, IndexT currentIndex, size_t offset_x, size_t offset_y) - { - image::Image current_color = input; - image::Image current_mask = inputMask; - - for (int l = 1; l <= _levelOfInterest; l++) { - - aliceVision::image::Image buf(current_color.Width(), current_color.Height()); - aliceVision::image::Image next_color(current_color.Width() / 2, current_color.Height() / 2); - aliceVision::image::Image next_mask(current_color.Width() / 2, current_color.Height() / 2); - - convolveGaussian5x5(buf, current_color); - downscale(next_color, buf); - - for (int i = 0; i < next_mask.Height(); i++) { - int di = i * 2; - - for (int j = 0; j < next_mask.Width(); j++) { - int dj = j * 2; - - if (current_mask(di, dj) && current_mask(di, dj + 1) && current_mask(di + 1, dj) && current_mask(di + 1, dj + 1)) { - next_mask(i, j) = 255; - } - else { - next_mask(i, j) = 0; - } - } - } - - current_color = next_color; - current_mask = next_mask; - offset_x /= 2; - offset_y /= 2; - } - - return _graphcut->append(current_color, current_mask, currentIndex, offset_x, offset_y); - } - - bool process() { - - if (!_graphcut->process()) { - return false; - } - - image::Image current_labels = _graphcut->getLabels(); - - for (int l = _levelOfInterest - 1; l >= 0; l--) { - - int nw = current_labels.Width() * 2; - int nh = current_labels.Height() * 2; - if (l == 0) { - nw = _outputWidth; - nh = _outputHeight; - } - - aliceVision::image::Image next_label(nw, nh); - for (int i = 0; i < nh; i++) { - int hi = i / 2; - - for (int j = 0; j < nw; j++) { - int hj = j / 2; - - next_label(i, j) = current_labels(hi, hj); - } - } - - current_labels = next_label; - } - - _labels = current_labels; - - return true; - } - - const image::Image & getLabels() { - return _labels; - } - -private: - std::unique_ptr _graphcut; - image::Image _labels; - size_t _levelOfInterest; - size_t _outputWidth; - size_t _outputHeight; -}; - -class LaplacianCompositer : public Compositer -{ -public: - - LaplacianCompositer(size_t outputWidth, size_t outputHeight, size_t bands) : - Compositer(outputWidth, outputHeight), - _pyramid_panorama(outputWidth, outputHeight, bands), - _bands(bands) { - - } - - virtual bool append(const aliceVision::image::Image & color, const aliceVision::image::Image & inputMask, const aliceVision::image::Image & inputWeights, size_t offset_x, size_t offset_y) - { - /*Get smalles size*/ - size_t minsize = std::min(color.Height(), color.Width()); - - /* - Look for the smallest scale such that the image is not smaller than the - convolution window size. - minsize / 2^x = 5 - minsize / 5 = 2^x - x = log2(minsize/5) - */ - const float gaussian_filter_size = 5.0f; - size_t optimal_scale = size_t(floor(std::log2(double(minsize) / gaussian_filter_size))); - if (optimal_scale < _bands) { - ALICEVISION_LOG_ERROR("Decreasing scale !"); - return false; - } - - if (optimal_scale > _bands) { - _bands = optimal_scale; - _pyramid_panorama.augment(_bands); - } - - size_t new_offset_x, new_offset_y; - aliceVision::image::Image color_pot; - aliceVision::image::Image mask_pot; - aliceVision::image::Image weights_pot; - makeImagePyramidCompatible(color_pot, new_offset_x, new_offset_y, color, offset_x, offset_y, _bands); - makeImagePyramidCompatible(mask_pot, new_offset_x, new_offset_y, inputMask, offset_x, offset_y, _bands); - makeImagePyramidCompatible(weights_pot, new_offset_x, new_offset_y, inputWeights, offset_x, offset_y, _bands); - - - aliceVision::image::Image feathered; - feathering(feathered, color_pot, mask_pot); - - /*To log space for hdr*/ - for (int i = 0; i < feathered.Height(); i++) { - for (int j = 0; j < feathered.Width(); j++) { - - feathered(i, j).r() = std::log(std::max(1e-8f, feathered(i, j).r())); - feathered(i, j).g() = std::log(std::max(1e-8f, feathered(i, j).g())); - feathered(i, j).b() = std::log(std::max(1e-8f, feathered(i, j).b())); - } - } - - _pyramid_panorama.apply(feathered, mask_pot, weights_pot, new_offset_x, new_offset_y); - - return true; - } - - virtual bool terminate() { - - _pyramid_panorama.rebuild(_panorama); - - /*Go back to normal space from log space*/ - for (int i = 0; i < _panorama.Height(); i++) { - for (int j = 0; j < _panorama.Width(); j++) { - _panorama(i, j).r() = std::exp(_panorama(i, j).r()); - _panorama(i, j).g() = std::exp(_panorama(i, j).g()); - _panorama(i, j).b() = std::exp(_panorama(i, j).b()); - } - } - - return true; - } - -protected: - LaplacianPyramid _pyramid_panorama; - size_t _bands; -}; - int aliceVision_main(int argc, char **argv) { std::string sfmDataFilepath; @@ -2273,7 +185,7 @@ int aliceVision_main(int argc, char **argv) // Compute seams std::vector> viewsToDraw; - std::unique_ptr distanceseams(new DistanceSeams(panoramaSize.first, panoramaSize.second)); + std::unique_ptr wtaSeams(new WTASeams(panoramaSize.first, panoramaSize.second)); if (isMultiBand) { std::map>> indexed_by_scale; @@ -2301,7 +213,7 @@ int aliceVision_main(int argc, char **argv) image::Image weights; image::readImage(weightsPath, weights, image::EImageColorSpace::NO_CONVERSION); - distanceseams->append(mask, weights, viewIt.first, offsetX, offsetY); + wtaSeams->append(mask, weights, viewIt.first, offsetX, offsetY); /*Get smalles size*/ size_t minsize = std::min(mask.Height(), mask.Width()); @@ -2335,9 +247,9 @@ int aliceVision_main(int argc, char **argv) } /* Retrieve seams from distance tool */ - image::Image labels = distanceseams->getLabels(); - distanceseams.reset(); - distanceseams = nullptr; + image::Image labels = wtaSeams->getLabels(); + wtaSeams.reset(); + wtaSeams = nullptr; if (isMultiBand && useGraphCut) { diff --git a/src/software/pipeline/main_panoramaWarping.cpp b/src/software/pipeline/main_panoramaWarping.cpp index f9d67d4e7b..25501a15a3 100644 --- a/src/software/pipeline/main_panoramaWarping.cpp +++ b/src/software/pipeline/main_panoramaWarping.cpp @@ -274,21 +274,31 @@ int aliceVision_main(int argc, char** argv) } // round to the closest tiles - coarseBbox.snapToGrid(tileSize); + BoundingBox snappedCoarseBbox; + snappedCoarseBbox = coarseBbox; + snappedCoarseBbox.snapToGrid(tileSize); //Initialize bouding box for image BoundingBox globalBbox; #pragma omp parallel for - for (int y = 0; y < coarseBbox.height; y += tileSize) { - for (int x = 0; x < coarseBbox.width; x += tileSize) { + for (int y = 0; y < snappedCoarseBbox.height; y += tileSize) { + + for (int x = 0; x < snappedCoarseBbox.width; x += tileSize) { BoundingBox localBbox; - localBbox.left = x + coarseBbox.left; - localBbox.top = y + coarseBbox.top; + localBbox.left = x + snappedCoarseBbox.left; + localBbox.top = y + snappedCoarseBbox.top; localBbox.width = tileSize; localBbox.height = tileSize; + + + /*int stillToProcess = coarseBbox.width - localBbox.left; + if (stillToProcess < tileSize) { + localBbox.width = stillToProcess; + }*/ + // Prepare coordinates map CoordinatesMap map; if (!map.build(panoramaSize, camPose, *(intrinsic.get()), localBbox)) { @@ -304,9 +314,12 @@ int aliceVision_main(int argc, char** argv) // Update bounding box - coarseBbox = globalBbox; - coarseBbox.snapToGrid(tileSize); - if (coarseBbox.width <= 0 || coarseBbox.height <= 0) continue; + BoundingBox snappedGlobalBbox; + + // Once again, snap to grid + snappedGlobalBbox = globalBbox; + snappedGlobalBbox.snapToGrid(tileSize); + if (snappedGlobalBbox.width <= 0 || snappedGlobalBbox.height <= 0) continue; // Load image and convert it to linear colorspace @@ -317,10 +330,10 @@ int aliceVision_main(int argc, char** argv) // Load metadata and update for output oiio::ParamValueList metadata = image::readImageMetadata(imagePath); - metadata.push_back(oiio::ParamValue("AliceVision:offsetX", coarseBbox.left)); - metadata.push_back(oiio::ParamValue("AliceVision:offsetY", coarseBbox.top)); - metadata.push_back(oiio::ParamValue("AliceVision:contentX", globalBbox.left - coarseBbox.left)); - metadata.push_back(oiio::ParamValue("AliceVision:contentY", globalBbox.top - coarseBbox.top)); + metadata.push_back(oiio::ParamValue("AliceVision:offsetX", snappedGlobalBbox.left)); + metadata.push_back(oiio::ParamValue("AliceVision:offsetY", snappedGlobalBbox.top)); + metadata.push_back(oiio::ParamValue("AliceVision:contentX", globalBbox.left - snappedGlobalBbox.left)); + metadata.push_back(oiio::ParamValue("AliceVision:contentY", globalBbox.top - snappedGlobalBbox.top)); metadata.push_back(oiio::ParamValue("AliceVision:contentW", globalBbox.width)); metadata.push_back(oiio::ParamValue("AliceVision:contentH", globalBbox.height)); metadata.push_back(oiio::ParamValue("AliceVision:panoramaWidth", panoramaSize.first)); @@ -342,9 +355,9 @@ int aliceVision_main(int argc, char** argv) std::unique_ptr out_weights = oiio::ImageOutput::create(weightFilepath); // Define output properties - oiio::ImageSpec spec_view(coarseBbox.width, coarseBbox.height, 3, (storageDataType == image::EStorageDataType::Half)?oiio::TypeDesc::HALF:oiio::TypeDesc::FLOAT); - oiio::ImageSpec spec_mask(coarseBbox.width, coarseBbox.height, 1, oiio::TypeDesc::UCHAR); - oiio::ImageSpec spec_weights(coarseBbox.width, coarseBbox.height, 1, oiio::TypeDesc::HALF); + oiio::ImageSpec spec_view(snappedGlobalBbox.width, snappedGlobalBbox.height, 3, (storageDataType == image::EStorageDataType::Half)?oiio::TypeDesc::HALF:oiio::TypeDesc::FLOAT); + oiio::ImageSpec spec_mask(snappedGlobalBbox.width, snappedGlobalBbox.height, 1, oiio::TypeDesc::UCHAR); + oiio::ImageSpec spec_weights(snappedGlobalBbox.width, snappedGlobalBbox.height, 1, oiio::TypeDesc::HALF); spec_view.tile_width = tileSize; spec_view.tile_height = tileSize; @@ -371,43 +384,60 @@ int aliceVision_main(int argc, char** argv) #pragma omp parallel for { - for (int y = 0; y < coarseBbox.height; y += tileSize) + for (int y = 0; y < snappedGlobalBbox.height; y += tileSize) { - for (int x = 0; x < coarseBbox.width; x += tileSize) + for (int x = 0; x < snappedGlobalBbox.width; x += tileSize) + { + BoundingBox localBbox; + localBbox.left = x + snappedGlobalBbox.left; + localBbox.top = y + snappedGlobalBbox.top; + localBbox.width = tileSize; + localBbox.height = tileSize; + + /*bool fillup = false; + int stillToProcess = globalBbox.width - localBbox.left; + if (stillToProcess < tileSize) { + to continue + fillup = true; + }*/ + + // Prepare coordinates map + CoordinatesMap map; + if (!map.build(panoramaSize, camPose, *(intrinsic.get()), localBbox)) + { + continue; + } + + // Warp image + GaussianWarper warper; + if (!warper.warp(map, pyramid)) { + continue; + } + + // Alpha mask + aliceVision::image::Image weights; + if (!distanceToCenter(weights, map, intrinsic->w(), intrinsic->h())) { + continue; + } + + // Store + #pragma omp critical + { + out_view->write_tile(x, y, 0, oiio::TypeDesc::FLOAT, warper.getColor().data()); + } + + // Store + #pragma omp critical { - BoundingBox localBbox; - localBbox.left = x + coarseBbox.left; - localBbox.top = y + coarseBbox.top; - localBbox.width = tileSize; - localBbox.height = tileSize; - - // Prepare coordinates map - CoordinatesMap map; - if (!map.build(panoramaSize, camPose, *(intrinsic.get()), localBbox)) - { - continue; - } - - // Warp image - GaussianWarper warper; - if (!warper.warp(map, pyramid)) { - continue; - } - - // Alpha mask - aliceVision::image::Image weights; - if (!distanceToCenter(weights, map, intrinsic->w(), intrinsic->h())) { - continue; - } - - // Store - #pragma omp critical - { - out_view->write_tile(x, y, 0, oiio::TypeDesc::FLOAT, warper.getColor().data()); - out_mask->write_tile(x, y, 0, oiio::TypeDesc::UCHAR, warper.getMask().data()); - out_weights->write_tile(x, y, 0, oiio::TypeDesc::FLOAT, weights.data()); - } + out_mask->write_tile(x, y, 0, oiio::TypeDesc::UCHAR, warper.getMask().data()); } + + // Store + #pragma omp critical + { + out_weights->write_tile(x, y, 0, oiio::TypeDesc::FLOAT, weights.data()); + } + } } } From 0438480387cbae7073ca7afce897e45a4bb9bb72 Mon Sep 17 00:00:00 2001 From: Fabien Servant Date: Fri, 2 Oct 2020 13:27:10 +0200 Subject: [PATCH 04/79] compositing for large images --- src/aliceVision/image/CMakeLists.txt | 2 + src/aliceVision/image/cache.cpp | 404 ++++++++++++++ src/aliceVision/image/cache.hpp | 333 ++++++++++++ src/aliceVision/panorama/alphaCompositer.hpp | 78 +-- src/aliceVision/panorama/boundingBox.hpp | 11 + src/aliceVision/panorama/cachedImage.hpp | 365 +++++++++++++ src/aliceVision/panorama/compositer.hpp | 185 ++++++- src/aliceVision/panorama/seams.cpp | 80 ++- src/aliceVision/panorama/seams.hpp | 21 +- .../pipeline/main_panoramaCompositing.cpp | 508 ++++++------------ .../pipeline/main_panoramaWarping.cpp | 1 + 11 files changed, 1579 insertions(+), 409 deletions(-) create mode 100644 src/aliceVision/image/cache.cpp create mode 100644 src/aliceVision/image/cache.hpp create mode 100644 src/aliceVision/panorama/cachedImage.hpp diff --git a/src/aliceVision/image/CMakeLists.txt b/src/aliceVision/image/CMakeLists.txt index 33d1cb08b2..531ec861f3 100644 --- a/src/aliceVision/image/CMakeLists.txt +++ b/src/aliceVision/image/CMakeLists.txt @@ -16,6 +16,7 @@ set(image_files_headers warping.hpp pixelTypes.hpp Sampler.hpp + cache.hpp ) # Sources @@ -23,6 +24,7 @@ set(image_files_sources convolution.cpp filtering.cpp io.cpp + cache.cpp ) alicevision_add_library(aliceVision_image diff --git a/src/aliceVision/image/cache.cpp b/src/aliceVision/image/cache.cpp new file mode 100644 index 0000000000..e1383fa63b --- /dev/null +++ b/src/aliceVision/image/cache.cpp @@ -0,0 +1,404 @@ +#include "cache.hpp" +#include + +namespace aliceVision +{ +namespace image +{ + +CacheManager::CacheManager(const std::string & pathStorage, size_t blockSize, size_t maxBlocksPerIndex) : +_basePathStorage(pathStorage), +_blockSize(blockSize), +_incoreBlockUsageCount(0), +_incoreBlockUsageMax(10), +_blockCountPerIndex(maxBlocksPerIndex) +{ + wipe(); +} + +CacheManager::~CacheManager() { + wipe(); +} + +void CacheManager::wipe() { + deleteIndexFiles(); + + _mru.clear(); + + _incoreBlockUsageCount = 0; + _nextStartBlockId = 0; + _nextObjectId = 0; +} + +void CacheManager::setInCoreMaxObjectCount(size_t max) { + _incoreBlockUsageMax = max; +} + +std::string CacheManager::getPathForIndex(size_t indexId) { + + if (_indexPaths.find(indexId) == _indexPaths.end()) { + + boost::filesystem::path path(_basePathStorage); + path /= boost::filesystem::unique_path(); + path += ".idx"; + + _indexPaths[indexId] = path.string(); + } + + return _indexPaths[indexId]; +} + +void CacheManager::deleteIndexFiles() { + + /* Remove all cache files */ + for (std::pair & p : _indexPaths) { + + boost::filesystem::path path(p.second); + boost::filesystem::remove(path); + } + + /* Remove list of cache files */ + _indexPaths.clear(); +} + +bool CacheManager::prepareBlockGroup(size_t startBlockId, size_t blocksCount) { + + size_t index_id = startBlockId / _blockCountPerIndex; + size_t block_id_in_index = startBlockId % _blockCountPerIndex; + size_t position_in_index = block_id_in_index * _blockSize; + size_t len = _blockSize * blocksCount; + + std::string pathname = getPathForIndex(index_id); + boost::filesystem::path path(pathname); + + std::ofstream file_index; + if (boost::filesystem::exists(path)) { + file_index.open(pathname, std::ios::binary | std::ios::out | std::ios::in); + } + else { + file_index.open(pathname, std::ios::binary | std::ios::out); + } + + if (!file_index.is_open()) { + return false; + } + + /*write a dummy byte at the end of the tile to "book" this place on disk*/ + file_index.seekp(position_in_index + len - 1, file_index.beg); + if (!file_index) { + return false; + } + + char c[1]; + c[0] = 0xff; + file_index.write(c, 1); + + return true; +} + +std::unique_ptr CacheManager::load(size_t startBlockId, size_t blockCount) { + + size_t indexId = startBlockId / _blockCountPerIndex; + size_t blockIdInIndex = startBlockId % _blockCountPerIndex; + size_t positionInIndex = blockIdInIndex * _blockSize; + size_t groupLength = _blockSize * blockCount; + + std::string path = getPathForIndex(indexId); + + std::ifstream file_index(path, std::ios::binary); + if (!file_index.is_open()) { + return std::unique_ptr(); + } + + file_index.seekg(positionInIndex, std::ios::beg); + if (file_index.fail()) { + return std::unique_ptr(); + } + + std::unique_ptr data(new unsigned char[groupLength]); + file_index.read(reinterpret_cast(data.get()), groupLength); + if (!file_index) { + return std::unique_ptr(); + } + + return data; +} + +bool CacheManager::save(std::unique_ptr && data, size_t startBlockId, size_t blockCount) { + + size_t indexId = startBlockId / _blockCountPerIndex; + size_t blockIdInIndex = startBlockId % _blockCountPerIndex; + size_t positionInIndex = blockIdInIndex * _blockSize; + size_t groupLength = _blockSize * blockCount; + + std::string path = getPathForIndex(indexId); + + std::ofstream file_index(path, std::ios::binary | std::ios::out | std::ios::in); + if (!file_index.is_open()) { + return false; + } + + file_index.seekp(positionInIndex, std::ios::beg); + if (file_index.fail()) { + return false; + } + + const unsigned char * bytesToWrite = data.get(); + if (bytesToWrite == nullptr) { + return false; + } + else { + /*Write data*/ + file_index.write(reinterpret_cast(bytesToWrite), groupLength); + } + + if (!file_index) { + return false; + } + + file_index.close(); + + return true; +} + +size_t CacheManager::getFreeBlockId(size_t blockCount) { + + size_t ret; + std::list & freeBlocksForCount = _freeBlocks[blockCount]; + + if (freeBlocksForCount.empty()) { + ret = _nextStartBlockId; + _nextStartBlockId += blockCount; + } + else { + ret = freeBlocksForCount.front(); + freeBlocksForCount.pop_front(); + } + + return ret; +} + +bool CacheManager::createObject(size_t & objectId, size_t blockCount) { + + objectId = _nextObjectId; + _nextObjectId++; + + MemoryItem item; + item.startBlockId = ~0; + item.countBlock = blockCount; + _memoryMap[objectId] = item; + + return true; +} + +bool CacheManager::acquireObject(std::unique_ptr & data, size_t objectId) { + + MemoryMap::iterator itfind = _memoryMap.find(objectId); + if (itfind == _memoryMap.end()) { + return false; + } + + MemoryItem memitem = itfind->second; + MRUItem item; + item.objectId = objectId; + item.objectSize = memitem.countBlock; + + /* Check mur */ + std::pair p = _mru.push_front(item); + if (p.second) { + + /* + Effectively added to the mru. + This means that we have to find this in the storage + */ + if (memitem.startBlockId == ~0) { + std::unique_ptr buffer(new unsigned char[_blockSize * memitem.countBlock]); + data = std::move(buffer); + } + else { + data = std::move(load(memitem.startBlockId, memitem.countBlock)); + } + + /*Update memory usage*/ + _incoreBlockUsageCount += memitem.countBlock; + } + else { + /* + The uid is present in the mru, put it in first position. + Note that the item may contain a previously deleted info + */ + _mru.relocate(_mru.begin(), p.first); + } + + while (_incoreBlockUsageCount > _incoreBlockUsageMax && _mru.size() > 1) { + + MRUItem item = _mru.back(); + + /*Remove item from mru*/ + _mru.pop_back(); + + /*Update memory usage*/ + _incoreBlockUsageCount -= item.objectSize; + + onRemovedFromMRU(item.objectId); + } + + + + return true; +} + +bool CacheManager::saveObject(std::unique_ptr && data, size_t objectId) { + + MemoryMap::iterator itfind = _memoryMap.find(objectId); + if (itfind == _memoryMap.end()) { + return false; + } + + MemoryItem item = itfind->second; + + if (itfind->second.startBlockId == ~0) { + + item.startBlockId = getFreeBlockId(item.countBlock); + _memoryMap[objectId] = item; + + prepareBlockGroup(item.startBlockId, item.countBlock); + } + + if (!save(std::move(data), item.startBlockId, item.countBlock)) { + return false; + } + + return true; +} + +void CacheManager::addFreeBlock(size_t blockId, size_t blockCount) { + + _freeBlocks[blockCount].push_back(blockId); +} + +size_t CacheManager::getActiveBlocks() const { + return _memoryMap.size(); +} + +CachedTile::~CachedTile() { + + std::shared_ptr manager = _manager.lock(); + if (manager) { + manager->notifyDestroy(_uid); + } +} + +bool CachedTile::acquire() { + + std::shared_ptr manager = _manager.lock(); + if (!manager) { + return false; + } + + + return manager->acquire(_uid); + + return true; +} + +TileCacheManager::TileCacheManager(const std::string & path_storage, size_t tileWidth, size_t tileHeight, size_t maxTilesPerIndex) : +CacheManager(path_storage, tileWidth * tileHeight, maxTilesPerIndex), +_tileWidth(tileWidth), _tileHeight(tileHeight) +{ +} + +std::shared_ptr TileCacheManager::create(const std::string & path_storage, size_t tileWidth, size_t tileHeight, size_t maxTilesPerIndex) { + + TileCacheManager * obj = new TileCacheManager(path_storage, tileWidth, tileHeight, maxTilesPerIndex); + + return std::shared_ptr(obj); +} + +std::shared_ptr TileCacheManager::requireNewCachedTile(size_t width, size_t height, size_t blockCount) { + + + CachedTile::smart_pointer ret; + size_t uid; + + if (!CacheManager::createObject(uid, blockCount)) { + return ret; + } + + /* Create container */ + std::shared_ptr sptr = shared_from_this(); + ret.reset(new CachedTile(sptr, uid, _tileWidth, _tileHeight, width, height, blockCount)); + + /*Store weak pointer internally*/ + _objectMap[uid] = ret; + + return ret; +} + +void TileCacheManager::notifyDestroy(size_t tileId) { + + /* Remove weak pointer */ + _objectMap.erase(tileId); + + /* Remove map from object to block id*/ + MemoryMap::iterator it = _memoryMap.find(tileId); + if (it == _memoryMap.end()) { + return; + } + size_t blockId = it->second.startBlockId; + size_t blockCount = it->second.countBlock; + _memoryMap.erase(it); + + /*If memory block is valid*/ + if (blockId != ~0) { + + /*Add block to list of available*/ + addFreeBlock(blockId, blockCount); + } +} + +bool TileCacheManager::acquire(size_t tileId) { + + + MapCachedTile::iterator itfind = _objectMap.find(tileId); + if (itfind == _objectMap.end()) { + return false; + } + + CachedTile::smart_pointer tile = itfind->second.lock(); + if (!tile) { + return false; + } + + /*Acquire the object*/ + std::unique_ptr content = tile->getData(); + if (!CacheManager::acquireObject(content, tileId)) { + return false; + } + + /*Update tile data*/ + tile->setData(std::move(content)); + + + return true; +} + +void TileCacheManager::onRemovedFromMRU(size_t objectId) { + + MapCachedTile::iterator itfind = _objectMap.find(objectId); + if (itfind == _objectMap.end()) { + return; + } + + CachedTile::smart_pointer tile = itfind->second.lock(); + if (!tile) { + return; + } + + /* Save object and set the tile data to nullptr */ + std::unique_ptr content = tile->getData(); + CacheManager::saveObject(std::move(content), objectId); +} + +} +} \ No newline at end of file diff --git a/src/aliceVision/image/cache.hpp b/src/aliceVision/image/cache.hpp new file mode 100644 index 0000000000..c53cd1239f --- /dev/null +++ b/src/aliceVision/image/cache.hpp @@ -0,0 +1,333 @@ +// This file is part of the AliceVision project. +// Copyright (c) 2016 AliceVision contributors. +// Copyright (c) 2012 openMVG contributors. +// This Source Code Form is subject to the terms of the Mozilla Public License, +// v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at https://mozilla.org/MPL/2.0/. + +#pragma once + +#include "aliceVision/numeric/numeric.hpp" +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +namespace aliceVision +{ +namespace image +{ + + +class TileCacheManager; + +/** + * A cached tile is an object containing a rectangular region of an image + * It has a tile width and a tile height. This is the memory used by this tile + * It also has a required width and a required height. This is the really used part of the tile + * This is because a tile may be on a border of the image and not fully used. + * It contains a pointer to its data which may be null if the data is out of core + */ +class CachedTile { +public: + using weak_pointer = std::weak_ptr; + using smart_pointer = std::shared_ptr; + +public: + + CachedTile() = delete; + + CachedTile(const std::shared_ptr & manager, size_t uid, size_t tileWidth, size_t tileHeight, size_t width, size_t height, size_t depth) + : _manager(manager),_uid(uid), _tileWidth(tileWidth), _tileHeight(tileHeight), _requiredWidth(width), _requiredHeight(height), _depth(depth) { + + /*Make sure the required size is less or equal than the tile size*/ + _requiredWidth = std::min(_requiredWidth, _tileWidth); + _requiredHeight = std::min(_requiredHeight, _tileHeight); + } + + ~CachedTile(); + + size_t getUid() const { + return _uid; + } + + size_t getTileWidth() const { + return _tileWidth; + } + + size_t getTileHeight() const { + return _tileHeight; + } + + size_t getRequiredWidth() const { + return _requiredWidth; + } + + size_t getRequiredHeight() const { + return _requiredHeight; + } + + size_t getDepth() const { + return _depth; + } + + /* + Tells the system that we need the data for this tile. + This means that is the data is out of core, we want it back + @return false if the process failed to grab data. + */ + bool acquire(); + + /** + * Update data with a new buffer + * Move the data parameter to the _data property. + * @note the parameter is invalidated ! + */ + void setData(std::unique_ptr && data) { + _data = std::move(data); + } + + /** + * Get a pointer to the contained data + * @return nullptr if the data is cached + */ + unsigned char * getDataPointer() const { + + if (!_data) { + return nullptr; + } + + return _data.get(); + } + + /** + * Move the data. + * The data is returned and the object property is set to nullptr + */ + std::unique_ptr getData() { + return std::move(_data); + } + +private: + std::unique_ptr _data = nullptr; + std::weak_ptr _manager; + + size_t _uid; + size_t _tileWidth; + size_t _tileHeight; + size_t _requiredWidth; + size_t _requiredHeight; + size_t _depth; +}; + +/* +An abstract concept of cache management for generic objects +*/ +class CacheManager { +public: + using IndexedStoragePaths = std::unordered_map; + using IndexedFreeBlocks = std::unordered_map>; + + /* + An item of the Most Recently used container + */ + struct MRUItem + { + size_t objectId; + size_t objectSize; + }; + + /* + An item of the object to memory block associative array + */ + struct MemoryItem + { + size_t startBlockId; + size_t countBlock; + }; + + using MemoryMap = std::map; + + /** + Most recently used object container + Used to know which objects have not been used for some time + */ + using MRUType = boost::multi_index::multi_index_container< + MRUItem, + boost::multi_index::indexed_by< + boost::multi_index::sequenced<>, + boost::multi_index::hashed_unique< + boost::multi_index::member< + MRUItem, + size_t, + &MRUItem::objectId> + > + > + >; + +public: + CacheManager() = delete; + + /** + * Create a cache manager + * Each created object is associated to this manager. + * @param pathStorage the path to the directory where the file will be stored + * @param blockSize the base size of an object + * @param maxTilesPerIndex the maximal number of blocks for a given file (give a maximal size for a cache file) + */ + CacheManager(const std::string & pathStorage, size_t blockSize, size_t maxBlocksPerIndex); + virtual ~CacheManager(); + + /** + * Set the maximal number number of items simultaneously in core + * @param max the maximal number of items + */ + void setInCoreMaxObjectCount(size_t max); + + /** + * Create a new object of size block count + * @param objectId the created object index + * @param blockCount the required size for this object (In number of blocks) + * @return true if the object was created + */ + bool createObject(size_t & objectId, size_t blockCount); + + /** + * Acquire a given object + * @param data the result data acquired + * @param objectId the object index to acquire + * @return true if the object was acquired + */ + bool acquireObject(std::unique_ptr & data, size_t objectId); + + /** + * Get the number of managed blocks + * @return a block count + */ + size_t getActiveBlocks() const; + +protected: + + std::string getPathForIndex(size_t indexId); + void deleteIndexFiles(); + void wipe(); + + bool prepareBlockGroup(size_t startBlockId, size_t blocksCount); + std::unique_ptr load(size_t startBlockId, size_t blocksCount); + bool save(std::unique_ptr && data, size_t startBlockId, size_t blockCount); + bool saveObject(std::unique_ptr && data, size_t objectId); + + virtual void onRemovedFromMRU(size_t objectId) = 0; + + void addFreeBlock(size_t blockId, size_t blockCount); + size_t getFreeBlockId(size_t blockCount); + + + +protected: + size_t _blockSize; + size_t _incoreBlockUsageCount; + size_t _incoreBlockUsageMax; + size_t _blockCountPerIndex; + size_t _nextStartBlockId; + size_t _nextObjectId; + + std::string _basePathStorage; + IndexedStoragePaths _indexPaths; + IndexedFreeBlocks _freeBlocks; + + MRUType _mru; + MemoryMap _memoryMap; +}; + +/** + * A cache manager specialized for image tiles + * All tiles in this image have a size multiple of a given base tile size. + */ +class TileCacheManager : public CacheManager, public std::enable_shared_from_this { +public: + using shared_ptr = std::shared_ptr; + using MapCachedTile = std::map; +public: + + TileCacheManager() = delete; + + /** + * There is no explicit constructor. + * We want to force the use of shared pointer for the manager + * @param pathStorage the path to the directory where the file will be stored + * @param tileWidth the base width of a tile + * @param tileWidth the base height of a tile + * @param maxTilesPerIndex the maximal number of tiles for a given file (give a maximal size for a cache file) + * @retuurn the manager shared pointer + */ + static std::shared_ptr create(const std::string & pathStorage, size_t tileWidth, size_t tileHeight, size_t maxTilesPerIndex); + + /** + * Notify the manager that a given tile was destroyed. + * @param tileId the tile index which was destroyed + * @note this method should not be explicitly called except by CachedTile + */ + void notifyDestroy(size_t tileId); + + /** + * Acquire a given tile + * @param tileId the tile index to acquire + * @return true if the tile was acquired + */ + bool acquire(size_t tileId); + + /** + * Acquire a given tile + * @param width the requested tile size (less or equal to the base tile size) + * @param height the requested tile size (less or equal to the base tile size) + * @param blockCount the requested tile size in depth + * @return a pointer to the cached tile created + */ + std::shared_ptr requireNewCachedTile(size_t width, size_t height, size_t blockCount); + + template + std::shared_ptr requireNewCachedTile(size_t width, size_t height) { + return requireNewCachedTile(width, height, sizeof(T)); + } + + /** + * Return the base tile width + * @return a width + */ + size_t getTileWidth() const { + return _tileWidth; + } + + /** + * Return the base tile height + * @return a width + */ + size_t getTileHeight() const { + return _tileHeight; + } + +protected: + + TileCacheManager(const std::string & path_storage, size_t tileWidth, size_t tileHeight, size_t maxTilesPerIndex); + + virtual void onRemovedFromMRU(size_t objectId); + +protected: + + size_t _tileWidth; + size_t _tileHeight; + + MapCachedTile _objectMap; +}; + +} +} \ No newline at end of file diff --git a/src/aliceVision/panorama/alphaCompositer.hpp b/src/aliceVision/panorama/alphaCompositer.hpp index 6bb3999ee1..fde368c1f1 100644 --- a/src/aliceVision/panorama/alphaCompositer.hpp +++ b/src/aliceVision/panorama/alphaCompositer.hpp @@ -8,75 +8,81 @@ namespace aliceVision class AlphaCompositer : public Compositer { public: - AlphaCompositer(size_t outputWidth, size_t outputHeight) - : Compositer(outputWidth, outputHeight) + AlphaCompositer(size_t outputWidth, size_t outputHeight) + : Compositer(outputWidth, outputHeight) { } virtual bool append(const aliceVision::image::Image& color, const aliceVision::image::Image& inputMask, - const aliceVision::image::Image& inputWeights, size_t offset_x, size_t offset_y) + const aliceVision::image::Image& inputWeights, int offset_x, int offset_y) { - for(size_t i = 0; i < color.Height(); i++) - { + aliceVision::image::Image masked(color.Width(), color.Height()); - size_t pano_i = offset_y + i; - if(pano_i >= _panorama.Height()) - { - continue; - } + BoundingBox panoramaBb; + panoramaBb.left = offset_x; + panoramaBb.top = offset_y; + panoramaBb.width = color.Width(); + panoramaBb.height = color.Height(); + + if (!loopyCachedImageExtract(masked, _panorama, panoramaBb)) + { + return false; + } + + for(size_t i = 0; i < color.Height(); i++) + { for(size_t j = 0; j < color.Width(); j++) { - if(!inputMask(i, j)) { continue; } - size_t pano_j = offset_x + j; - if(pano_j >= _panorama.Width()) - { - pano_j = pano_j - _panorama.Width(); - } - float wc = inputWeights(i, j); - _panorama(pano_i, pano_j).r() += wc * color(i, j).r(); - _panorama(pano_i, pano_j).g() += wc * color(i, j).g(); - _panorama(pano_i, pano_j).b() += wc * color(i, j).b(); - _panorama(pano_i, pano_j).a() += wc; + masked(i, j).r() += wc * color(i, j).r(); + masked(i, j).g() += wc * color(i, j).g(); + masked(i, j).b() += wc * color(i, j).b(); + masked(i, j).a() += wc; } } + if (!loopyCachedImageAssign(_panorama, masked, panoramaBb)) { + return false; + } + return true; } virtual bool terminate() { - - for(int i = 0; i < _panorama.Height(); i++) - { - for(int j = 0; j < _panorama.Width(); j++) + + _panorama.perPixelOperation( + [](image::RGBAfColor c) -> image::RGBAfColor { + image::RGBAfColor r; - if(_panorama(i, j).a() < 1e-6) + if (c.a() < 1e-6f) { - _panorama(i, j).r() = 1.0f; - _panorama(i, j).g() = 0.0f; - _panorama(i, j).b() = 0.0f; - _panorama(i, j).a() = 0.0f; + r.r() = 1.0f; + r.g() = 0.0f; + r.b() = 0.0f; + r.a() = 0.0f; } - else + else { - _panorama(i, j).r() = _panorama(i, j).r() / _panorama(i, j).a(); - _panorama(i, j).g() = _panorama(i, j).g() / _panorama(i, j).a(); - _panorama(i, j).b() = _panorama(i, j).b() / _panorama(i, j).a(); - _panorama(i, j).a() = 1.0f; + r.r() = c.r() / c.a(); + r.g() = c.g() / c.a(); + r.b() = c.b() / c.a(); + r.a() = 1.0f; } + + return r; } - } + ); return true; } diff --git a/src/aliceVision/panorama/boundingBox.hpp b/src/aliceVision/panorama/boundingBox.hpp index 5883c4f01c..0a14252227 100644 --- a/src/aliceVision/panorama/boundingBox.hpp +++ b/src/aliceVision/panorama/boundingBox.hpp @@ -71,4 +71,15 @@ struct BoundingBox width = maxr - left + 1; height = maxb - top + 1; } + + bool isInside(const BoundingBox& other) const { + + if (other.left > left) return false; + if (other.top > top) return false; + + if (other.getRight() < getRight()) return false; + if (other.getBottom() < getBottom()) return false; + + return true; + } }; \ No newline at end of file diff --git a/src/aliceVision/panorama/cachedImage.hpp b/src/aliceVision/panorama/cachedImage.hpp new file mode 100644 index 0000000000..4e3ef010b8 --- /dev/null +++ b/src/aliceVision/panorama/cachedImage.hpp @@ -0,0 +1,365 @@ +#pragma once + +#include +#include +#include +#include + +namespace aliceVision +{ + +template +class CachedImage +{ +public: + bool createImage(std::shared_ptr manager, size_t width, size_t height) + { + + _width = width; + _height = height; + _tileSize = manager->getTileWidth(); + + _tilesArray.clear(); + + int countHeight = int(ceil(double(height) / double(manager->getTileHeight()))); + int countWidth = int(ceil(double(width) / double(manager->getTileWidth()))); + + _memoryWidth = countWidth * _tileSize; + _memoryHeight = countHeight * _tileSize; + + for(int i = 0; i < countHeight; i++) + { + + int tile_height = manager->getTileHeight(); + if(i == countHeight - 1) + { + tile_height = height - (i * tile_height); + } + + std::vector row; + + for(int j = 0; j < countWidth; j++) + { + + int tile_width = manager->getTileWidth(); + if(j == countWidth - 1) + { + tile_width = width - (i * tile_width); + } + + image::CachedTile::smart_pointer tile = manager->requireNewCachedTile(tile_width, tile_height); + if(tile == nullptr) + { + return false; + } + + row.push_back(tile); + } + + _tilesArray.push_back(row); + } + + return true; + } + + bool writeImage(const std::string& path) + { + + ALICEVISION_LOG_ERROR("incorrect template function"); + return false; + } + + template + bool perPixelOperation(UnaryFunction f) + { + + for(int i = 0; i < _tilesArray.size(); i++) + { + + std::vector& row = _tilesArray[i]; + + for(int j = 0; j < _tilesArray[i].size(); j++) + { + + image::CachedTile::smart_pointer ptr = row[j]; + if(!ptr) + { + continue; + } + + if(!ptr->acquire()) + { + continue; + } + + T* data = (T*)ptr->getDataPointer(); + + std::transform(data, data + ptr->getTileWidth() * ptr->getTileHeight(), data, f); + } + } + + return true; + } + + bool assign(const aliceVision::image::Image& input, const BoundingBox & inputBb, const BoundingBox & outputBb) + { + BoundingBox outputMemoryBb; + outputMemoryBb.left = 0; + outputMemoryBb.top = 0; + outputMemoryBb.width = _width; + outputMemoryBb.height = _height; + + if(!outputBb.isInside(outputMemoryBb)) + { + return false; + } + + BoundingBox inputMemoryBb; + inputMemoryBb.left = 0; + inputMemoryBb.top = 0; + inputMemoryBb.width = input.Width(); + inputMemoryBb.height = input.Height(); + + if(!inputBb.isInside(inputMemoryBb)) + { + return false; + } + + BoundingBox snapedBb = outputBb; + snapedBb.snapToGrid(_tileSize); + + BoundingBox gridBb; + gridBb.left = snapedBb.left / _tileSize; + gridBb.top = snapedBb.top / _tileSize; + gridBb.width = snapedBb.width / _tileSize; + gridBb.height = snapedBb.height / _tileSize; + + for(int i = 0; i < gridBb.height; i++) + { + int ti = gridBb.top + i; + int cy = ti * _tileSize; + int sy = inputBb.top + i * _tileSize; + + int start_y = std::max(0, outputBb.top - cy); + int end_y = std::min(_tileSize - 1, outputBb.getBottom() - cy); + + std::vector& row = _tilesArray[ti]; + + for(int j = 0; j < gridBb.width; j++) + { + int tj = gridBb.left + j; + int cx = tj * _tileSize; + int sx = inputBb.left + j * _tileSize; + + int start_x = std::max(0, outputBb.left - cx); + int end_x = std::min(_tileSize - 1, outputBb.getRight() - cx); + + image::CachedTile::smart_pointer ptr = row[tj]; + if(!ptr) + { + continue; + } + + if(!ptr->acquire()) + { + continue; + } + + T* data = (T*)ptr->getDataPointer(); + + for(int y = start_y; y <= end_y; y++) + { + for(int x = start_x; x <= end_x; x++) + { + data[y * _tileSize + x] = input(sy + y, sx + x); + } + } + } + } + + return true; + } + + bool extract(aliceVision::image::Image& output, const BoundingBox & outputBb, const BoundingBox& inputBb) + { + BoundingBox thisBb; + thisBb.left = 0; + thisBb.top = 0; + thisBb.width = _width; + thisBb.height = _height; + + if(!inputBb.isInside(thisBb)) + { + return false; + } + + BoundingBox outputMemoryBb; + outputMemoryBb.left = 0; + outputMemoryBb.top = 0; + outputMemoryBb.width = output.Width(); + outputMemoryBb.height = output.Height(); + + if(!outputBb.isInside(outputMemoryBb)) + { + return false; + } + + + BoundingBox snapedBb = inputBb; + snapedBb.snapToGrid(_tileSize); + + BoundingBox gridBb; + gridBb.left = snapedBb.left / _tileSize; + gridBb.top = snapedBb.top / _tileSize; + gridBb.width = snapedBb.width / _tileSize; + gridBb.height = snapedBb.height / _tileSize; + + for(int i = 0; i < gridBb.height; i++) + { + int ti = gridBb.top + i; + int cy = ti * _tileSize; + int sy = outputBb.top + i * _tileSize; + + int start_y = std::max(0, inputBb.top - cy); + int end_y = std::min(_tileSize - 1, inputBb.getBottom() - cy); + + std::vector& row = _tilesArray[ti]; + + for(int j = 0; j < gridBb.width; j++) + { + int tj = gridBb.left + j; + int cx = tj * _tileSize; + int sx = outputBb.left + j * _tileSize; + + int start_x = std::max(0, inputBb.left - cx); + int end_x = std::min(_tileSize - 1, inputBb.getRight() - cx); + + image::CachedTile::smart_pointer ptr = row[tj]; + if(!ptr) + { + continue; + } + + if(!ptr->acquire()) + { + continue; + } + + T* data = (T*)ptr->getDataPointer(); + + for(int y = start_y; y <= end_y; y++) + { + for(int x = start_x; x <= end_x; x++) + { + output(sy + y, sx + x) = data[y * _tileSize + x]; + } + } + } + } + + return true; + } + + std::vector>& getTiles() { return _tilesArray; } + + int getWidth() const { return _width; } + + int getHeight() const { return _height; } + +private: + int _width; + int _height; + int _memoryWidth; + int _memoryHeight; + int _tileSize; + + std::vector> _tilesArray; +}; + +template <> +bool CachedImage::writeImage(const std::string& path) +{ + + std::unique_ptr out = oiio::ImageOutput::create(path); + if(!out) + { + return false; + } + + oiio::ImageSpec spec(_memoryWidth, _memoryHeight, 4, oiio::TypeDesc::FLOAT); + spec.tile_width = _tileSize; + spec.tile_height = _tileSize; + + if(!out->open(path, spec)) + { + return false; + } + + for(int i = 0; i < _tilesArray.size(); i++) + { + + std::vector& row = _tilesArray[i]; + + for(int j = 0; j < row.size(); j++) + { + + if(!row[j]->acquire()) + { + return false; + } + + unsigned char* ptr = row[j]->getDataPointer(); + + out->write_tile(j * _tileSize, i * _tileSize, 0, oiio::TypeDesc::FLOAT, ptr); + } + } + + out->close(); + + return true; +} + +template <> +bool CachedImage::writeImage(const std::string& path) +{ + + std::unique_ptr out = oiio::ImageOutput::create(path); + if(!out) + { + return false; + } + + oiio::ImageSpec spec(_memoryWidth, _memoryHeight, 1, oiio::TypeDesc::UINT32); + spec.tile_width = _tileSize; + spec.tile_height = _tileSize; + + if(!out->open(path, spec)) + { + return false; + } + + for(int i = 0; i < _tilesArray.size(); i++) + { + + std::vector& row = _tilesArray[i]; + + for(int j = 0; j < row.size(); j++) + { + + if(!row[j]->acquire()) + { + return false; + } + + unsigned char* ptr = row[j]->getDataPointer(); + + out->write_tile(j * _tileSize, i * _tileSize, 0, oiio::TypeDesc::UINT32, ptr); + } + } + + out->close(); + + return true; +} + +} // namespace aliceVision \ No newline at end of file diff --git a/src/aliceVision/panorama/compositer.hpp b/src/aliceVision/panorama/compositer.hpp index c7aeaef010..2ca45b87b3 100644 --- a/src/aliceVision/panorama/compositer.hpp +++ b/src/aliceVision/panorama/compositer.hpp @@ -2,61 +2,198 @@ #include +#include "cachedImage.hpp" + namespace aliceVision { +template +bool loopyCachedImageAssign(CachedImage & output, const aliceVision::image::Image & input, const BoundingBox & assignedOutputBb) +{ + BoundingBox inputBb; + BoundingBox outputBb; + + inputBb.left = 0; + inputBb.top = 0; + inputBb.width = input.Width(); + inputBb.height = input.Height(); + outputBb = assignedOutputBb; + + if (assignedOutputBb.getRight() < output.getWidth()) { + + if (!output.assign(input, inputBb, outputBb)) + { + return false; + } + } + else { + + int left_1 = assignedOutputBb.left; + int left_2 = 0; + int width1 = output.getWidth() - assignedOutputBb.left; + int width2 = input.Width() - width1; + + inputBb.left = 0; + outputBb.left = left_1; + inputBb.width = width1; + outputBb.width = width1; + + if (!output.assign(input, inputBb, outputBb)) + { + return false; + } + + inputBb.left = width1; + outputBb.left = 0; + inputBb.width = width2; + outputBb.width = width2; + + if (!output.assign(input, inputBb, outputBb)) + { + return false; + } + } + + return true; +} + +template +bool loopyCachedImageExtract(aliceVision::image::Image & output, CachedImage & input, const BoundingBox & extractedInputBb) +{ + BoundingBox outputBb; + BoundingBox inputBb; + + outputBb.left = 0; + outputBb.top = 0; + outputBb.width = output.Width(); + outputBb.height = output.Height(); + inputBb = extractedInputBb; + + if (extractedInputBb.getRight() < input.getWidth()) + { + if (!input.extract(output, outputBb, inputBb)) { + return false; + } + } + else + { + int left_1 = extractedInputBb.left; + int left_2 = 0; + int width_1 = input.getWidth() - extractedInputBb.left; + int width_2 = output.Width() - width_1; + + outputBb.left = 0; + inputBb.left = left_1; + outputBb.width = width_1; + inputBb.width = width_1; + + if (!input.extract(output, outputBb, inputBb)) + { + return false; + } + + outputBb.left = width_1; + inputBb.left = 0; + outputBb.width = width_2; + inputBb.width = width_2; + + if (!input.extract(output, outputBb, inputBb)) + { + return false; + } + } + + return true; +} + class Compositer { public: - Compositer(size_t outputWidth, size_t outputHeight) - : _panorama(outputWidth, outputHeight, true, image::RGBAfColor(0.0f, 0.0f, 0.0f, 0.0f)) + Compositer(int width, int height) : + _panoramaWidth(width), + _panoramaHeight(height) { + } virtual bool append(const aliceVision::image::Image& color, const aliceVision::image::Image& inputMask, - const aliceVision::image::Image& inputWeights, size_t offset_x, size_t offset_y) + const aliceVision::image::Image& inputWeights, int offset_x, int offset_y) { + aliceVision::image::Image masked(color.Width(), color.Height()); - for(size_t i = 0; i < color.Height(); i++) - { + BoundingBox panoramaBb; + panoramaBb.left = offset_x; + panoramaBb.top = offset_y; + panoramaBb.width = color.Width(); + panoramaBb.height = color.Height(); - size_t pano_i = offset_y + i; - if(pano_i >= _panorama.Height()) - { - continue; - } + if (!loopyCachedImageExtract(masked, _panorama, panoramaBb)) + { + return false; + } + + for(size_t i = 0; i < color.Height(); i++) + { for(size_t j = 0; j < color.Width(); j++) { - if(!inputMask(i, j)) { continue; } - size_t pano_j = offset_x + j; - if(pano_j >= _panorama.Width()) - { - pano_j = pano_j - _panorama.Width(); - } - - _panorama(pano_i, pano_j).r() = color(i, j).r(); - _panorama(pano_i, pano_j).g() = color(i, j).g(); - _panorama(pano_i, pano_j).b() = color(i, j).b(); - _panorama(pano_i, pano_j).a() = 1.0f; + masked(i, j).r() = color(i, j).r(); + masked(i, j).g() = color(i, j).g(); + masked(i, j).b() = color(i, j).b(); + masked(i, j).a() = 1.0f; } } + if (!loopyCachedImageAssign(_panorama, masked, panoramaBb)) { + return false; + } + return true; } + virtual bool initialize(image::TileCacheManager::shared_ptr & cacheManager) { + + if(!_panorama.createImage(cacheManager, _panoramaWidth, _panoramaHeight)) + { + return false; + } + + if(!_panorama.perPixelOperation( + [](image::RGBAfColor ) -> image::RGBAfColor + { + return image::RGBAfColor(0.0f, 0.0f, 0.0f, 0.0f); + }) + ) + { + return false; + } + + return true; + } + virtual bool terminate() { return true; } - aliceVision::image::Image& getPanorama() { return _panorama; } + bool save(const std::string &path) + { + if(!_panorama.writeImage(path)) + { + return false; + } + + return true; + } protected: - aliceVision::image::Image _panorama; + + CachedImage _panorama; + int _panoramaWidth; + int _panoramaHeight; }; -} // namespace aliceVision \ No newline at end of file +} // namespace aliceVision diff --git a/src/aliceVision/panorama/seams.cpp b/src/aliceVision/panorama/seams.cpp index bcb4f7c333..91ba2bd578 100644 --- a/src/aliceVision/panorama/seams.cpp +++ b/src/aliceVision/panorama/seams.cpp @@ -2,6 +2,7 @@ #include "gaussian.hpp" #include "imageOps.hpp" +#include "compositer.hpp" namespace aliceVision { @@ -135,42 +136,93 @@ void drawSeams(aliceVision::image::Image& inout, aliceVision: } } +bool WTASeams::initialize(image::TileCacheManager::shared_ptr & cacheManager) +{ + if(!_weights.createImage(cacheManager, _panoramaWidth, _panoramaHeight)) + { + return false; + } + + if(!_weights.perPixelOperation( + [](float ) -> float + { + return 0.0f; + })) + { + return false; + } + + if(!_labels.createImage(cacheManager, _panoramaWidth, _panoramaHeight)) + { + return false; + } + + if(!_labels.perPixelOperation( + [](IndexT ) -> IndexT + { + return UndefinedIndexT; + })) + { + return false; + } + + return true; +} + bool WTASeams::append(const aliceVision::image::Image& inputMask, const aliceVision::image::Image& inputWeights, IndexT currentIndex, size_t offset_x, size_t offset_y) { - if(inputMask.size() != inputWeights.size()) + if( inputMask.size() != inputWeights.size()) { return false; } - for(int i = 0; i < inputMask.Height(); i++) + aliceVision::image::Image weights(inputMask.Width(), inputMask.Height()); + aliceVision::image::Image labels(inputMask.Width(), inputMask.Height()); + + BoundingBox globalBb; + globalBb.left = offset_x; + globalBb.top = offset_y; + globalBb.width = inputMask.Width(); + globalBb.height = inputMask.Height(); + + if (!loopyCachedImageExtract(weights, _weights, globalBb)) { + return false; + } - int di = i + offset_y; + if (!loopyCachedImageExtract(labels, _labels, globalBb)) + { + return false; + } + - for(int j = 0; j < inputMask.Width(); j++) + for(size_t i = 0; i < weights.Height(); i++) + { + for(size_t j = 0; j < weights.Width(); j++) { - if(!inputMask(i, j)) { continue; } - int dj = j + offset_x; - if(dj >= _weights.Width()) + if (inputWeights(i, j) > weights(i, j)) { - dj = dj - _weights.Width(); - } - - if(inputWeights(i, j) > _weights(di, dj)) - { - _labels(di, dj) = currentIndex; - _weights(di, dj) = inputWeights(i, j); + labels(i, j) = currentIndex; + weights(i, j) = inputWeights(i, j); } } } + if (!loopyCachedImageAssign(_weights, weights, globalBb)) { + return false; + } + + if (!loopyCachedImageAssign(_labels, labels, globalBb)) { + return false; + } + return true; } diff --git a/src/aliceVision/panorama/seams.hpp b/src/aliceVision/panorama/seams.hpp index 57ab7e958b..22e4795748 100644 --- a/src/aliceVision/panorama/seams.hpp +++ b/src/aliceVision/panorama/seams.hpp @@ -3,6 +3,7 @@ #include #include +#include "cachedImage.hpp" #include "graphcut.hpp" namespace aliceVision @@ -18,22 +19,30 @@ class WTASeams { public: WTASeams(size_t outputWidth, size_t outputHeight) - : _weights(outputWidth, outputHeight, true, 0.0f) - , _labels(outputWidth, outputHeight, true, 255) + : _panoramaWidth(outputWidth) + , _panoramaHeight(outputHeight) { } virtual ~WTASeams() = default; - virtual bool append(const aliceVision::image::Image& inputMask, + bool initialize(image::TileCacheManager::shared_ptr & cacheManager); + + bool append(const aliceVision::image::Image& inputMask, const aliceVision::image::Image& inputWeights, IndexT currentIndex, size_t offset_x, size_t offset_y); - const image::Image& getLabels() { return _labels; } + CachedImage & getLabels() + { + return _labels; + } private: - image::Image _weights; - image::Image _labels; + CachedImage _weights; + CachedImage _labels; + + int _panoramaWidth; + int _panoramaHeight; }; class HierarchicalGraphcutSeams diff --git a/src/software/pipeline/main_panoramaCompositing.cpp b/src/software/pipeline/main_panoramaCompositing.cpp index c61c5d5e9b..b94812ed00 100644 --- a/src/software/pipeline/main_panoramaCompositing.cpp +++ b/src/software/pipeline/main_panoramaCompositing.cpp @@ -27,19 +27,11 @@ #include #include -#include -#include -#include -#include -#include -#include -#include +#include #include #include -#include #include - // These constants define the current software version. // They must be updated when the command line is changed. #define ALICEVISION_SOFTWARE_VERSION_MAJOR 1 @@ -51,361 +43,219 @@ namespace po = boost::program_options; namespace bpt = boost::property_tree; namespace fs = boost::filesystem; -int aliceVision_main(int argc, char **argv) +bool computeWTALabels(CachedImage & labels, image::TileCacheManager::shared_ptr& cacheManager, const sfmData::SfMData& sfmData, const std::string & inputPath, std::pair & panoramaSize) { - std::string sfmDataFilepath; - std::string warpingFolder; - std::string outputPanorama; - - std::string compositerType = "multiband"; - std::string overlayType = "none"; - bool useGraphCut = true; - bool showBorders = false; - bool showSeams = false; - - image::EStorageDataType storageDataType = image::EStorageDataType::Float; - - system::EVerboseLevel verboseLevel = system::Logger::getDefaultVerboseLevel(); - - // Program description - po::options_description allParams ( - "Perform panorama stiching of cameras around a nodal point for 360° panorama creation. \n" - "AliceVision PanoramaCompositing" - ); - - // Description of mandatory parameters - po::options_description requiredParams("Required parameters"); - requiredParams.add_options() - ("input,i", po::value(&sfmDataFilepath)->required(), "Input sfmData.") - ("warpingFolder,w", po::value(&warpingFolder)->required(), "Folder with warped images.") - ("output,o", po::value(&outputPanorama)->required(), "Path of the output panorama."); - allParams.add(requiredParams); - - // Description of optional parameters - po::options_description optionalParams("Optional parameters"); - optionalParams.add_options() - ("compositerType,c", po::value(&compositerType)->required(), "Compositer Type [replace, alpha, multiband].") - ("overlayType,c", po::value(&overlayType)->required(), "Overlay Type [none, borders, seams, all].") - ("useGraphCut,c", po::value(&useGraphCut)->default_value(useGraphCut), "Do we use graphcut for ghost removal ?") - ("storageDataType", po::value(&storageDataType)->default_value(storageDataType), - ("Storage data type: " + image::EStorageDataType_informations()).c_str()); - allParams.add(optionalParams); - - // Setup log level given command line - po::options_description logParams("Log parameters"); - logParams.add_options() - ("verboseLevel,v", po::value(&verboseLevel)->default_value(verboseLevel), "verbosity level (fatal, error, warning, info, debug, trace)."); - allParams.add(logParams); - - - // Effectively parse command line given parse options - po::variables_map vm; - try - { - po::store(po::parse_command_line(argc, argv, allParams), vm); - - if(vm.count("help") || (argc == 1)) - { - ALICEVISION_COUT(allParams); - return EXIT_SUCCESS; - } - po::notify(vm); - } - catch(boost::program_options::required_option& e) - { - ALICEVISION_CERR("ERROR: " << e.what()); - ALICEVISION_COUT("Usage:\n\n" << allParams); - return EXIT_FAILURE; - } - catch(boost::program_options::error& e) - { - ALICEVISION_CERR("ERROR: " << e.what()); - ALICEVISION_COUT("Usage:\n\n" << allParams); - return EXIT_FAILURE; - } - - ALICEVISION_COUT("Program called with the following parameters:"); - ALICEVISION_COUT(vm); - - // Set verbose level given command line - system::Logger::get()->setLogLevel(verboseLevel); - - if (overlayType == "borders" || overlayType == "all") - { - showBorders = true; - } - - if (overlayType == "seams" || overlayType == "all") { - showSeams = true; - } - - // load input scene - sfmData::SfMData sfmData; - if(!sfmDataIO::Load(sfmData, sfmDataFilepath, sfmDataIO::ESfMData(sfmDataIO::VIEWS|sfmDataIO::EXTRINSICS|sfmDataIO::INTRINSICS))) - { - ALICEVISION_LOG_ERROR("The input file '" + sfmDataFilepath + "' cannot be read"); - return EXIT_FAILURE; - } - - std::pair panoramaSize; - { - const IndexT viewId = *sfmData.getValidViews().begin(); - const std::string viewFilepath = (fs::path(warpingFolder) / (std::to_string(viewId) + ".exr")).string(); - ALICEVISION_LOG_TRACE("Read panorama size from file: " << viewFilepath); - - oiio::ParamValueList metadata = image::readImageMetadata(viewFilepath); - panoramaSize.first = metadata.find("AliceVision:panoramaWidth")->get_int(); - panoramaSize.second = metadata.find("AliceVision:panoramaHeight")->get_int(); - - if(panoramaSize.first == 0 || panoramaSize.second == 0) - { - ALICEVISION_LOG_ERROR("The output panorama size is empty."); - return EXIT_FAILURE; - } - ALICEVISION_LOG_INFO("Output panorama size set to " << panoramaSize.first << "x" << panoramaSize.second); - } - - std::unique_ptr compositer; - bool isMultiBand = false; - if (compositerType == "multiband") - { - compositer = std::unique_ptr(new LaplacianCompositer(panoramaSize.first, panoramaSize.second, 1)); - isMultiBand = true; - } - else if (compositerType == "alpha") - { - compositer = std::unique_ptr(new AlphaCompositer(panoramaSize.first, panoramaSize.second)); - } - else - { - compositer = std::unique_ptr(new Compositer(panoramaSize.first, panoramaSize.second)); - } - - - // Compute seams - std::vector> viewsToDraw; - - std::unique_ptr wtaSeams(new WTASeams(panoramaSize.first, panoramaSize.second)); - if (isMultiBand) - { - std::map>> indexed_by_scale; - for (const auto& viewIt : sfmData.getViews()) + WTASeams seams(panoramaSize.first, panoramaSize.second); + + if (!seams.initialize(cacheManager)) { - if(!sfmData.isPoseAndIntrinsicDefined(viewIt.second.get())) - { - // skip unreconstructed views - continue; - } - - // Load mask - const std::string maskPath = (fs::path(warpingFolder) / (std::to_string(viewIt.first) + "_mask.exr")).string(); - ALICEVISION_LOG_INFO("Load mask with path " << maskPath); - image::Image mask; - image::readImage(maskPath, mask, image::EImageColorSpace::NO_CONVERSION); - - oiio::ParamValueList metadata = image::readImageMetadata(maskPath); - const std::size_t offsetX = metadata.find("AliceVision:offsetX")->get_int(); - const std::size_t offsetY = metadata.find("AliceVision:offsetY")->get_int(); - - // Load Weights - const std::string weightsPath = (fs::path(warpingFolder) / (std::to_string(viewIt.first) + "_weight.exr")).string(); - ALICEVISION_LOG_INFO("Load weights with path " << weightsPath); - image::Image weights; - image::readImage(weightsPath, weights, image::EImageColorSpace::NO_CONVERSION); - - wtaSeams->append(mask, weights, viewIt.first, offsetX, offsetY); - - /*Get smalles size*/ - size_t minsize = std::min(mask.Height(), mask.Width()); - - /* - minsize / 2^x = 8 - minsize / 8 = 2^x - x = log2(minsize/8) - */ - size_t optimal_scale = size_t(floor(std::log2(double(minsize) / 5.0))); - indexed_by_scale[optimal_scale].push_back(viewIt.second); + return false; } - for (auto item : indexed_by_scale) { - for (auto view : item.second) { - viewsToDraw.push_back(view); - } - } - } - else { - for (auto& viewIt : sfmData.getViews()) + for(const auto& viewIt : sfmData.getViews()) { - if(!sfmData.isPoseAndIntrinsicDefined(viewIt.second.get())) - { - // skip unreconstructed views - continue; - } - - viewsToDraw.push_back(viewIt.second); - } - } - - /* Retrieve seams from distance tool */ - image::Image labels = wtaSeams->getLabels(); - wtaSeams.reset(); - wtaSeams = nullptr; - - if (isMultiBand && useGraphCut) { - - int initial_level = 0; - int max_width_for_graphcut = 5000; - double ratio = double(panoramaSize.first) / double(max_width_for_graphcut); - if (ratio > 1.0) { - initial_level = int(ceil(log2(ratio))); - } - - for (int l = initial_level; l>= 0; l--) { - HierarchicalGraphcutSeams seams(panoramaSize.first, panoramaSize.second, l); - seams.setOriginalLabels(labels); - if (l != initial_level) { - seams.setMaximalDistance(100); - } - - for (const auto& viewIt : sfmData.getViews()) - { if(!sfmData.isPoseAndIntrinsicDefined(viewIt.second.get())) { // skip unreconstructed views continue; } - + // Load mask - const std::string maskPath = (fs::path(warpingFolder) / (std::to_string(viewIt.first) + "_mask.exr")).string(); + const std::string maskPath = (fs::path(inputPath) / (std::to_string(viewIt.first) + "_mask.exr")).string(); ALICEVISION_LOG_INFO("Load mask with path " << maskPath); image::Image mask; image::readImage(maskPath, mask, image::EImageColorSpace::NO_CONVERSION); + // Get offset oiio::ParamValueList metadata = image::readImageMetadata(maskPath); const std::size_t offsetX = metadata.find("AliceVision:offsetX")->get_int(); const std::size_t offsetY = metadata.find("AliceVision:offsetY")->get_int(); - // Load Color - const std::string colorsPath = (fs::path(warpingFolder) / (std::to_string(viewIt.first) + ".exr")).string(); - ALICEVISION_LOG_INFO("Load colors with path " << colorsPath); - image::Image colors; - image::readImage(colorsPath, colors, image::EImageColorSpace::NO_CONVERSION); - - seams.append(colors, mask, viewIt.first, offsetX, offsetY); - } - - if (seams.process()) { - ALICEVISION_LOG_INFO("Updating labels with graphcut"); - labels = seams.getLabels(); - } + // Load Weights + const std::string weightsPath = (fs::path(inputPath) / (std::to_string(viewIt.first) + "_weight.exr")).string(); + ALICEVISION_LOG_INFO("Load weights with path " << weightsPath); + image::Image weights; + image::readImage(weightsPath, weights, image::EImageColorSpace::NO_CONVERSION); + + seams.append(mask, weights, viewIt.first, offsetX, offsetY); } - } - oiio::ParamValueList outputMetadata; + labels = seams.getLabels(); - // Do compositing - for (const auto & view : viewsToDraw) - { - IndexT viewId = view->getViewId(); + return true; +} - if(!sfmData.isPoseAndIntrinsicDefined(view.get())) +int aliceVision_main(int argc, char** argv) +{ + std::string sfmDataFilepath; + std::string warpingFolder; + std::string outputPanorama; + + std::string compositerType = "multiband"; + std::string overlayType = "none"; + bool useGraphCut = true; + bool showBorders = false; + bool showSeams = false; + + image::EStorageDataType storageDataType = image::EStorageDataType::Float; + + system::EVerboseLevel verboseLevel = system::Logger::getDefaultVerboseLevel(); + + // Program description + po::options_description allParams( + "Perform panorama stiching of cameras around a nodal point for 360° panorama creation. \n" + "AliceVision PanoramaCompositing"); + + // Description of mandatory parameters + po::options_description requiredParams("Required parameters"); + requiredParams.add_options()("input,i", po::value(&sfmDataFilepath)->required(), "Input sfmData.")( + "warpingFolder,w", po::value(&warpingFolder)->required(), "Folder with warped images.")( + "output,o", po::value(&outputPanorama)->required(), "Path of the output panorama."); + allParams.add(requiredParams); + + // Description of optional parameters + po::options_description optionalParams("Optional parameters"); + optionalParams.add_options()("compositerType,c", po::value(&compositerType)->required(), + "Compositer Type [replace, alpha, multiband].")( + "overlayType,c", po::value(&overlayType)->required(), "Overlay Type [none, borders, seams, all].")( + "useGraphCut,c", po::value(&useGraphCut)->default_value(useGraphCut), + "Do we use graphcut for ghost removal ?")( + "storageDataType", po::value(&storageDataType)->default_value(storageDataType), + ("Storage data type: " + image::EStorageDataType_informations()).c_str()); + allParams.add(optionalParams); + + // Setup log level given command line + po::options_description logParams("Log parameters"); + logParams.add_options()("verboseLevel,v", + po::value(&verboseLevel)->default_value(verboseLevel), + "verbosity level (fatal, error, warning, info, debug, trace)."); + allParams.add(logParams); + + // Effectively parse command line given parse options + po::variables_map vm; + try { - // skip unreconstructed views - continue; + po::store(po::parse_command_line(argc, argv, allParams), vm); + + if(vm.count("help") || (argc == 1)) + { + ALICEVISION_COUT(allParams); + return EXIT_SUCCESS; + } + po::notify(vm); } + catch(boost::program_options::required_option& e) + { + ALICEVISION_CERR("ERROR: " << e.what()); + ALICEVISION_COUT("Usage:\n\n" << allParams); + return EXIT_FAILURE; + } + catch(boost::program_options::error& e) + { + ALICEVISION_CERR("ERROR: " << e.what()); + ALICEVISION_COUT("Usage:\n\n" << allParams); + return EXIT_FAILURE; + } + + ALICEVISION_COUT("Program called with the following parameters:"); + ALICEVISION_COUT(vm); - // Load image and convert it to linear colorspace - const std::string imagePath = (fs::path(warpingFolder) / (std::to_string(viewId) + ".exr")).string(); - ALICEVISION_LOG_INFO("Load image with path " << imagePath); - image::Image source; - image::readImage(imagePath, source, image::EImageColorSpace::NO_CONVERSION); + // Set verbose level given command line + system::Logger::get()->setLogLevel(verboseLevel); - oiio::ParamValueList metadata = image::readImageMetadata(imagePath); - if(outputMetadata.empty()) + // load input scene + sfmData::SfMData sfmData; + if(!sfmDataIO::Load(sfmData, sfmDataFilepath, + sfmDataIO::ESfMData(sfmDataIO::VIEWS | sfmDataIO::EXTRINSICS | sfmDataIO::INTRINSICS))) { - // the first one will define the output metadata (random selection) - outputMetadata = metadata; + ALICEVISION_LOG_ERROR("The input file '" + sfmDataFilepath + "' cannot be read"); + return EXIT_FAILURE; } - const std::size_t offsetX = metadata.find("AliceVision:offsetX")->get_int(); - const std::size_t offsetY = metadata.find("AliceVision:offsetY")->get_int(); - - // Load mask - const std::string maskPath = (fs::path(warpingFolder) / (std::to_string(viewId) + "_mask.exr")).string(); - ALICEVISION_LOG_INFO("Load mask with path " << maskPath); - image::Image mask; - image::readImage(maskPath, mask, image::EImageColorSpace::NO_CONVERSION); - - // Load Weights - const std::string weightsPath = (fs::path(warpingFolder) / (std::to_string(viewId) + "_weight.exr")).string(); - ALICEVISION_LOG_INFO("Load weights with path " << weightsPath); - image::Image weights; - image::readImage(weightsPath, weights, image::EImageColorSpace::NO_CONVERSION); - - // Build weight map - if (isMultiBand) + + int tileSize; + std::pair panoramaSize; { - image::Image seams(weights.Width(), weights.Height()); - getMaskFromLabels(seams, labels, viewId, offsetX, offsetY); + const IndexT viewId = *sfmData.getValidViews().begin(); + const std::string viewFilepath = (fs::path(warpingFolder) / (std::to_string(viewId) + ".exr")).string(); + ALICEVISION_LOG_TRACE("Read panorama size from file: " << viewFilepath); - // Composite image into panorama - compositer->append(source, mask, seams, offsetX, offsetY); + oiio::ParamValueList metadata = image::readImageMetadata(viewFilepath); + panoramaSize.first = metadata.find("AliceVision:panoramaWidth")->get_int(); + panoramaSize.second = metadata.find("AliceVision:panoramaHeight")->get_int(); + tileSize = metadata.find("AliceVision:tileSize")->get_int(); + + if(panoramaSize.first == 0 || panoramaSize.second == 0) + { + ALICEVISION_LOG_ERROR("The output panorama size is empty."); + return EXIT_FAILURE; + } + + if(tileSize == 0) + { + ALICEVISION_LOG_ERROR("no information on tileSize"); + return EXIT_FAILURE; + } + + ALICEVISION_LOG_INFO("Output panorama size set to " << panoramaSize.first << "x" << panoramaSize.second); } - else + + // Create a cache manager + const std::string outputPath = boost::filesystem::path(outputPanorama).parent_path().string(); + image::TileCacheManager::shared_ptr cacheManager = image::TileCacheManager::create(outputPath, 256, 256, 65536); + if(!cacheManager) { - compositer->append(source, mask, weights, offsetX, offsetY); + ALICEVISION_LOG_ERROR("Error creating the cache manager"); + return EXIT_FAILURE; } - } - // Build image - compositer->terminate(); - + // Configure the cache manager memory + cacheManager->setInCoreMaxObjectCount(1000); + + AlphaCompositer compositer(panoramaSize.first, panoramaSize.second); + compositer.initialize(cacheManager); - if (showBorders) - { - for (const auto& viewIt : sfmData.getViews()) + CachedImage labels; + computeWTALabels(labels, cacheManager, sfmData, warpingFolder, panoramaSize); + + /*for(const auto& it : sfmData.getViews()) { - if(!sfmData.isPoseAndIntrinsicDefined(viewIt.second.get())) - { - // skip unreconstructed views - continue; - } - - // Load mask - const std::string maskPath = (fs::path(warpingFolder) / (std::to_string(viewIt.first) + "_mask.exr")).string(); - ALICEVISION_LOG_INFO("Load mask with path " << maskPath); - image::Image mask; - image::readImage(maskPath, mask, image::EImageColorSpace::NO_CONVERSION); - - oiio::ParamValueList metadata = image::readImageMetadata(maskPath); - const std::size_t offsetX = metadata.find("AliceVision:offsetX")->get_int(); - const std::size_t offsetY = metadata.find("AliceVision:offsetY")->get_int(); - - drawBorders(compositer->getPanorama(), mask, offsetX, offsetY); - } - } - - if (showSeams) - { - drawSeams(compositer->getPanorama(), labels); - } - - // Remove Warping-specific metadata - outputMetadata.remove("AliceVision:offsetX"); - outputMetadata.remove("AliceVision:offsetY"); - outputMetadata.remove("AliceVision:panoramaWidth"); - outputMetadata.remove("AliceVision:panoramaHeight"); - // no notion of extra orientation on the output panorama - outputMetadata.remove("Orientation"); - outputMetadata.remove("orientation"); - - // Store output - ALICEVISION_LOG_INFO("Write output panorama to file " << outputPanorama); - const aliceVision::image::Image & panorama = compositer->getPanorama(); - - // Select storage data type - outputMetadata.push_back(oiio::ParamValue("AliceVision:storageDataType", image::EStorageDataType_enumToString(storageDataType))); - - image::writeImage(outputPanorama, panorama, image::EImageColorSpace::AUTO, outputMetadata); - - return EXIT_SUCCESS; + IndexT viewId = it.first; + auto view = it.second; + + if(!sfmData.isPoseAndIntrinsicDefined(view.get())) + { + // skip unreconstructed views + continue; + } + + // Load image and convert it to linear colorspace + const std::string imagePath = (fs::path(warpingFolder) / (std::to_string(viewId) + ".exr")).string(); + ALICEVISION_LOG_INFO("Load image with path " << imagePath); + image::Image source; + image::readImage(imagePath, source, image::EImageColorSpace::NO_CONVERSION); + + oiio::ParamValueList metadata = image::readImageMetadata(imagePath); + const std::size_t offsetX = metadata.find("AliceVision:offsetX")->get_int(); + const std::size_t offsetY = metadata.find("AliceVision:offsetY")->get_int(); + + // Load mask + const std::string maskPath = (fs::path(warpingFolder) / (std::to_string(viewId) + "_mask.exr")).string(); + ALICEVISION_LOG_INFO("Load mask with path " << maskPath); + image::Image mask; + image::readImage(maskPath, mask, image::EImageColorSpace::NO_CONVERSION); + + // Load Weights + const std::string weightsPath = (fs::path(warpingFolder) / (std::to_string(viewId) + "_weight.exr")).string(); + ALICEVISION_LOG_INFO("Load weights with path " << weightsPath); + image::Image weights; + image::readImage(weightsPath, weights, image::EImageColorSpace::NO_CONVERSION); + + compositer.append(source, mask, weights, offsetX, offsetY); + }*/ + + compositer.terminate(); + + //compositer.save(outputPanorama); + + return EXIT_SUCCESS; } diff --git a/src/software/pipeline/main_panoramaWarping.cpp b/src/software/pipeline/main_panoramaWarping.cpp index 25501a15a3..e8e0cee06e 100644 --- a/src/software/pipeline/main_panoramaWarping.cpp +++ b/src/software/pipeline/main_panoramaWarping.cpp @@ -338,6 +338,7 @@ int aliceVision_main(int argc, char** argv) metadata.push_back(oiio::ParamValue("AliceVision:contentH", globalBbox.height)); metadata.push_back(oiio::ParamValue("AliceVision:panoramaWidth", panoramaSize.first)); metadata.push_back(oiio::ParamValue("AliceVision:panoramaHeight", panoramaSize.second)); + metadata.push_back(oiio::ParamValue("AliceVision:tileSize", tileSize)); // Images will be converted in Panorama coordinate system, so there will be no more extra orientation. metadata.remove("Orientation"); From 78aa0559ef4f6039f6d7003bd3acbe8be861f4de Mon Sep 17 00:00:00 2001 From: Fabien Servant Date: Sat, 3 Oct 2020 19:31:20 +0200 Subject: [PATCH 05/79] WIP on laplacian compositing for large images --- src/aliceVision/image/Image.hpp | 10 + src/aliceVision/panorama/CMakeLists.txt | 1 + src/aliceVision/panorama/cachedImage.cpp | 135 ++++++++++ src/aliceVision/panorama/cachedImage.hpp | 249 ++++++++++-------- src/aliceVision/panorama/compositer.hpp | 15 +- src/aliceVision/panorama/coordinatesMap.cpp | 2 - src/aliceVision/panorama/graphcut.hpp | 185 +++++++------ src/aliceVision/panorama/imageOps.cpp | 26 +- src/aliceVision/panorama/imageOps.hpp | 3 +- .../panorama/laplacianCompositer.hpp | 57 ++-- src/aliceVision/panorama/laplacianPyramid.cpp | 150 +++++++---- src/aliceVision/panorama/laplacianPyramid.hpp | 18 +- src/aliceVision/panorama/seams.cpp | 62 +++-- src/aliceVision/panorama/seams.hpp | 29 +- .../pipeline/main_panoramaCompositing.cpp | 106 +++++++- 15 files changed, 735 insertions(+), 313 deletions(-) create mode 100644 src/aliceVision/panorama/cachedImage.cpp diff --git a/src/aliceVision/image/Image.hpp b/src/aliceVision/image/Image.hpp index 893456d038..948c826c63 100644 --- a/src/aliceVision/image/Image.hpp +++ b/src/aliceVision/image/Image.hpp @@ -222,7 +222,17 @@ namespace aliceVision */ template< typename T1> friend Image operator-( const Image & imgA , const Image & imgB ) ; + + template + bool perPixelOperation(UnaryFunction f) + { + for(auto row : this->rowwise()) + { + std::transform(row.begin(), row.end(), row.begin(), f); + } + return true; + } protected : //-- Image data are stored by inheritance of a matrix diff --git a/src/aliceVision/panorama/CMakeLists.txt b/src/aliceVision/panorama/CMakeLists.txt index dfa5123554..5d48cfaa4a 100644 --- a/src/aliceVision/panorama/CMakeLists.txt +++ b/src/aliceVision/panorama/CMakeLists.txt @@ -16,6 +16,7 @@ set(panorama_files_sources laplacianPyramid.cpp seams.cpp imageOps.cpp + cachedImage.cpp ) alicevision_add_library(aliceVision_panorama diff --git a/src/aliceVision/panorama/cachedImage.cpp b/src/aliceVision/panorama/cachedImage.cpp new file mode 100644 index 0000000000..d36e50177c --- /dev/null +++ b/src/aliceVision/panorama/cachedImage.cpp @@ -0,0 +1,135 @@ +#include "cachedImage.hpp" + +namespace aliceVision +{ + +template <> +bool CachedImage::writeImage(const std::string& path) +{ + + std::unique_ptr out = oiio::ImageOutput::create(path); + if(!out) + { + return false; + } + + oiio::ImageSpec spec(_memoryWidth, _memoryHeight, 4, oiio::TypeDesc::FLOAT); + spec.tile_width = _tileSize; + spec.tile_height = _tileSize; + + if(!out->open(path, spec)) + { + return false; + } + + for(int i = 0; i < _tilesArray.size(); i++) + { + + std::vector& row = _tilesArray[i]; + + for(int j = 0; j < row.size(); j++) + { + + if(!row[j]->acquire()) + { + return false; + } + + unsigned char* ptr = row[j]->getDataPointer(); + + out->write_tile(j * _tileSize, i * _tileSize, 0, oiio::TypeDesc::FLOAT, ptr); + } + } + + out->close(); + + return true; +} + +template <> +bool CachedImage::writeImage(const std::string& path) +{ + + std::unique_ptr out = oiio::ImageOutput::create(path); + if(!out) + { + return false; + } + + oiio::ImageSpec spec(_memoryWidth, _memoryHeight, 3, oiio::TypeDesc::FLOAT); + spec.tile_width = _tileSize; + spec.tile_height = _tileSize; + + if(!out->open(path, spec)) + { + return false; + } + + for(int i = 0; i < _tilesArray.size(); i++) + { + + std::vector& row = _tilesArray[i]; + + for(int j = 0; j < row.size(); j++) + { + + if(!row[j]->acquire()) + { + return false; + } + + unsigned char* ptr = row[j]->getDataPointer(); + + out->write_tile(j * _tileSize, i * _tileSize, 0, oiio::TypeDesc::FLOAT, ptr); + } + } + + out->close(); + + return true; +} + +template <> +bool CachedImage::writeImage(const std::string& path) +{ + + std::unique_ptr out = oiio::ImageOutput::create(path); + if(!out) + { + return false; + } + + oiio::ImageSpec spec(_memoryWidth, _memoryHeight, 1, oiio::TypeDesc::UINT32); + spec.tile_width = _tileSize; + spec.tile_height = _tileSize; + + if(!out->open(path, spec)) + { + return false; + } + + for(int i = 0; i < _tilesArray.size(); i++) + { + + std::vector& row = _tilesArray[i]; + + for(int j = 0; j < row.size(); j++) + { + + if(!row[j]->acquire()) + { + return false; + } + + unsigned char* ptr = row[j]->getDataPointer(); + + out->write_tile(j * _tileSize, i * _tileSize, 0, oiio::TypeDesc::UINT32, ptr); + } + } + + out->close(); + + return true; +} + +} // namespace aliceVision \ No newline at end of file diff --git a/src/aliceVision/panorama/cachedImage.hpp b/src/aliceVision/panorama/cachedImage.hpp index 4e3ef010b8..8da1fb2193 100644 --- a/src/aliceVision/panorama/cachedImage.hpp +++ b/src/aliceVision/panorama/cachedImage.hpp @@ -4,6 +4,7 @@ #include #include #include +#include namespace aliceVision { @@ -101,6 +102,100 @@ class CachedImage return true; } + template + bool perPixelOperation(CachedImage & other, BinaryFunction f) + { + if (other.getWidth() != _width || other.getHeight() != _height) + { + return false; + } + + for(int i = 0; i < _tilesArray.size(); i++) + { + std::vector& row = _tilesArray[i]; + std::vector& rowOther = other.getTiles()[i]; + + for(int j = 0; j < _tilesArray[i].size(); j++) + { + + image::CachedTile::smart_pointer ptr = row[j]; + if(!ptr) + { + continue; + } + + image::CachedTile::smart_pointer ptrOther = rowOther[j]; + if(!ptrOther) + { + continue; + } + + if(!ptr->acquire()) + { + continue; + } + + if(!ptrOther->acquire()) + { + continue; + } + + T* data = (T*)ptr->getDataPointer(); + T2* dataOther = (T2*)ptrOther->getDataPointer(); + + std::transform(data, data + ptr->getTileWidth() * ptr->getTileHeight(), dataOther, data, f); + } + } + + return true; + } + + bool deepCopy(CachedImage & source) + { + if (source._memoryWidth != _memoryWidth) return false; + if (source._memoryHeight != _memoryHeight) return false; + if (source._tileSize != _tileSize) return false; + + for(int i = 0; i < _tilesArray.size(); i++) + { + std::vector & row = _tilesArray[i]; + std::vector & rowSource = source._tilesArray[i]; + + for(int j = 0; j < _tilesArray[i].size(); j++) + { + + image::CachedTile::smart_pointer ptr = row[j]; + if(!ptr) + { + continue; + } + + image::CachedTile::smart_pointer ptrSource = rowSource[j]; + if(!ptrSource) + { + continue; + } + + if (!ptr->acquire()) + { + continue; + } + + if (!ptrSource->acquire()) + { + continue; + } + + T * data = (T*)ptr->getDataPointer(); + T * dataSource = (T*)ptrSource->getDataPointer(); + + std::memcpy(data, dataSource, _tileSize * _tileSize * sizeof(T)); + } + } + + return true; + } + bool assign(const aliceVision::image::Image& input, const BoundingBox & inputBb, const BoundingBox & outputBb) { BoundingBox outputMemoryBb; @@ -125,34 +220,43 @@ class CachedImage return false; } + if (inputBb.width != outputBb.width) + { + return false; + } + + if (inputBb.height != outputBb.height) + { + return false; + } + + + //Make sure we have our bounding box aligned with the tiles BoundingBox snapedBb = outputBb; snapedBb.snapToGrid(_tileSize); + //Compute grid parameters BoundingBox gridBb; gridBb.left = snapedBb.left / _tileSize; gridBb.top = snapedBb.top / _tileSize; gridBb.width = snapedBb.width / _tileSize; gridBb.height = snapedBb.height / _tileSize; + int delta_y = outputBb.top - snapedBb.top; + int delta_x = outputBb.left - snapedBb.left; + for(int i = 0; i < gridBb.height; i++) { + //ibb.top + i * tileSize --> snapedBb.top + delta + i * tileSize int ti = gridBb.top + i; - int cy = ti * _tileSize; - int sy = inputBb.top + i * _tileSize; - - int start_y = std::max(0, outputBb.top - cy); - int end_y = std::min(_tileSize - 1, outputBb.getBottom() - cy); - + int sy = inputBb.top - delta_y + i * _tileSize; + std::vector& row = _tilesArray[ti]; for(int j = 0; j < gridBb.width; j++) { int tj = gridBb.left + j; - int cx = tj * _tileSize; - int sx = inputBb.left + j * _tileSize; - - int start_x = std::max(0, outputBb.left - cx); - int end_x = std::min(_tileSize - 1, outputBb.getRight() - cx); + int sx = inputBb.left - delta_x + j * _tileSize; image::CachedTile::smart_pointer ptr = row[tj]; if(!ptr) @@ -167,10 +271,12 @@ class CachedImage T* data = (T*)ptr->getDataPointer(); - for(int y = start_y; y <= end_y; y++) + for(int y = 0; y < _tileSize; y++) { - for(int x = start_x; x <= end_x; x++) + for(int x = 0; x < _tileSize; x++) { + if (sy + y < 0 || sy + y >= input.Height()) continue; + if (sx + x < 0 || sx + x >= input.Width()) continue; data[y * _tileSize + x] = input(sy + y, sx + x); } } @@ -204,6 +310,16 @@ class CachedImage return false; } + if (inputBb.width != outputBb.width) + { + return false; + } + + if (inputBb.height != outputBb.height) + { + return false; + } + BoundingBox snapedBb = inputBb; snapedBb.snapToGrid(_tileSize); @@ -214,25 +330,21 @@ class CachedImage gridBb.width = snapedBb.width / _tileSize; gridBb.height = snapedBb.height / _tileSize; + int delta_y = inputBb.top - snapedBb.top; + int delta_x = inputBb.left - snapedBb.left; + for(int i = 0; i < gridBb.height; i++) { int ti = gridBb.top + i; - int cy = ti * _tileSize; - int sy = outputBb.top + i * _tileSize; + int sy = outputBb.top - delta_y + i * _tileSize; - int start_y = std::max(0, inputBb.top - cy); - int end_y = std::min(_tileSize - 1, inputBb.getBottom() - cy); std::vector& row = _tilesArray[ti]; for(int j = 0; j < gridBb.width; j++) { int tj = gridBb.left + j; - int cx = tj * _tileSize; - int sx = outputBb.left + j * _tileSize; - - int start_x = std::max(0, inputBb.left - cx); - int end_x = std::min(_tileSize - 1, inputBb.getRight() - cx); + int sx = outputBb.left - delta_x + j * _tileSize; image::CachedTile::smart_pointer ptr = row[tj]; if(!ptr) @@ -247,10 +359,14 @@ class CachedImage T* data = (T*)ptr->getDataPointer(); - for(int y = start_y; y <= end_y; y++) + for(int y = 0; y < _tileSize; y++) { - for(int x = start_x; x <= end_x; x++) + for(int x = 0; x < _tileSize; x++) { + if (sy + y < 0 || sy + y >= output.Height()) continue; + if (sx + x < 0 || sx + x >= output.Width()) continue; + if (y < 0 || x < 0) continue; + output(sy + y, sx + x) = data[y * _tileSize + x]; } } @@ -277,89 +393,12 @@ class CachedImage }; template <> -bool CachedImage::writeImage(const std::string& path) -{ - - std::unique_ptr out = oiio::ImageOutput::create(path); - if(!out) - { - return false; - } - - oiio::ImageSpec spec(_memoryWidth, _memoryHeight, 4, oiio::TypeDesc::FLOAT); - spec.tile_width = _tileSize; - spec.tile_height = _tileSize; - - if(!out->open(path, spec)) - { - return false; - } - - for(int i = 0; i < _tilesArray.size(); i++) - { - - std::vector& row = _tilesArray[i]; - - for(int j = 0; j < row.size(); j++) - { - - if(!row[j]->acquire()) - { - return false; - } - - unsigned char* ptr = row[j]->getDataPointer(); - - out->write_tile(j * _tileSize, i * _tileSize, 0, oiio::TypeDesc::FLOAT, ptr); - } - } - - out->close(); - - return true; -} +bool CachedImage::writeImage(const std::string& path); template <> -bool CachedImage::writeImage(const std::string& path) -{ - - std::unique_ptr out = oiio::ImageOutput::create(path); - if(!out) - { - return false; - } - - oiio::ImageSpec spec(_memoryWidth, _memoryHeight, 1, oiio::TypeDesc::UINT32); - spec.tile_width = _tileSize; - spec.tile_height = _tileSize; +bool CachedImage::writeImage(const std::string& path); - if(!out->open(path, spec)) - { - return false; - } - - for(int i = 0; i < _tilesArray.size(); i++) - { - - std::vector& row = _tilesArray[i]; - - for(int j = 0; j < row.size(); j++) - { - - if(!row[j]->acquire()) - { - return false; - } - - unsigned char* ptr = row[j]->getDataPointer(); - - out->write_tile(j * _tileSize, i * _tileSize, 0, oiio::TypeDesc::UINT32, ptr); - } - } - - out->close(); - - return true; -} +template <> +bool CachedImage::writeImage(const std::string& path); } // namespace aliceVision \ No newline at end of file diff --git a/src/aliceVision/panorama/compositer.hpp b/src/aliceVision/panorama/compositer.hpp index 2ca45b87b3..5c62185d78 100644 --- a/src/aliceVision/panorama/compositer.hpp +++ b/src/aliceVision/panorama/compositer.hpp @@ -19,6 +19,12 @@ bool loopyCachedImageAssign(CachedImage & output, const aliceVision::image::I inputBb.height = input.Height(); outputBb = assignedOutputBb; + if (outputBb.getBottom() >= output.getHeight()) + { + outputBb.height = output.getHeight() - outputBb.top; + inputBb.height = outputBb.height; + } + if (assignedOutputBb.getRight() < output.getWidth()) { if (!output.assign(input, inputBb, outputBb)) @@ -67,11 +73,18 @@ bool loopyCachedImageExtract(aliceVision::image::Image & output, CachedImage< outputBb.top = 0; outputBb.width = output.Width(); outputBb.height = output.Height(); + inputBb = extractedInputBb; + if (inputBb.getBottom() >= input.getHeight()) + { + inputBb.height = input.getHeight() - inputBb.top; + outputBb.height = inputBb.height; + } if (extractedInputBb.getRight() < input.getWidth()) { - if (!input.extract(output, outputBb, inputBb)) { + if (!input.extract(output, outputBb, inputBb)) + { return false; } } diff --git a/src/aliceVision/panorama/coordinatesMap.cpp b/src/aliceVision/panorama/coordinatesMap.cpp index d20df974f0..acf2e86637 100644 --- a/src/aliceVision/panorama/coordinatesMap.cpp +++ b/src/aliceVision/panorama/coordinatesMap.cpp @@ -1,5 +1,3 @@ -#pragma once - #include "coordinatesMap.hpp" #include "sphericalMapping.hpp" diff --git a/src/aliceVision/panorama/graphcut.hpp b/src/aliceVision/panorama/graphcut.hpp index 3fdcdeb073..b366442ea2 100644 --- a/src/aliceVision/panorama/graphcut.hpp +++ b/src/aliceVision/panorama/graphcut.hpp @@ -6,6 +6,7 @@ #include #include "distance.hpp" +#include "boundingBox.hpp" namespace aliceVision { @@ -211,43 +212,60 @@ bool computeSeamsMap(image::Image& seams, const image::Image; using ImageOwners = image::Image>; public: GraphcutSeams(size_t outputWidth, size_t outputHeight) - : _owners(outputWidth, outputHeight, true) - , _labels(outputWidth, outputHeight, true, 0) - , _original_labels(outputWidth, outputHeight, true, 0) - , _distancesSeams(outputWidth, outputHeight, true, 0) + : _outputWidth(outputWidth), _outputHeight(outputHeight) , _maximal_distance_change(outputWidth + outputHeight) { } virtual ~GraphcutSeams() = default; - void setOriginalLabels(const image::Image& existing_labels) + void setOriginalLabels(CachedImage & existing_labels) { - _labels = existing_labels; - _original_labels = existing_labels; + _labels.deepCopy(existing_labels); + _original_labels.deepCopy(existing_labels); - image::Image seams(_labels.Width(), _labels.Height()); + /*image::Image seams(_labels.Width(), _labels.Height()); computeSeamsMap(seams, _labels); - computeDistanceMap(_distancesSeams, seams); + computeDistanceMap(_distancesSeams, seams);*/ + } + + bool initialize(image::TileCacheManager::shared_ptr & cacheManager) + { + if(!_labels.createImage(cacheManager, _outputWidth, _outputHeight)) + { + return false; + } + + if(!_original_labels.createImage(cacheManager, _outputWidth, _outputHeight)) + { + return false; + } + + if(!_distancesSeams.createImage(cacheManager, _outputWidth, _outputHeight)) + { + return false; + } + + if(!_distancesSeams.perPixelOperation( + [](int) -> int + { + return 0; + }) + ) + { + return false; + } + + return true; } - virtual bool append(const aliceVision::image::Image& input, - const aliceVision::image::Image& inputMask, IndexT currentIndex, size_t offset_x, - size_t offset_y) + bool append(const aliceVision::image::Image& input, const aliceVision::image::Image& inputMask, IndexT currentIndex, size_t offset_x, size_t offset_y) { if(inputMask.size() != input.size()) @@ -255,21 +273,21 @@ class GraphcutSeams return false; } - Rect rect; + BoundingBox rect; - rect.l = offset_x; - rect.t = offset_y; - rect.w = input.Width() + 1; - rect.h = input.Height() + 1; + rect.left = offset_x; + rect.top = offset_y; + rect.width = input.Width() + 1; + rect.height = input.Height() + 1; /*Extend rect for borders*/ - rect.l = std::max(0, rect.l - 3); - rect.t = std::max(0, rect.t - 3); - rect.w = rect.w + 6; - rect.h = rect.h + 6; - if(rect.t + rect.h > _owners.Height()) + rect.left = std::max(0, rect.left - 3); + rect.top = std::max(0, rect.top - 3); + rect.width = rect.width + 6; + rect.height = rect.height + 6; + if(rect.top + rect.height > _owners.Height()) { - rect.h = _owners.Height() - rect.t; + rect.height = _owners.Height() - rect.top; } _rects[currentIndex] = rect; @@ -324,13 +342,13 @@ class GraphcutSeams /*For each possible label, try to extends its domination on the label's world */ bool change = false; - for(auto& info : _rects) + for(auto & info : _rects) { ALICEVISION_LOG_INFO("Graphcut expansion (iteration " << i << ") for label " << info.first); - int p1 = info.second.l; - int w1 = info.second.w; + int p1 = info.second.left; + int w1 = info.second.width; int p2 = 0; int w2 = 0; @@ -338,13 +356,11 @@ class GraphcutSeams { w1 = _labels.Width() - p1; p2 = 0; - w2 = info.second.w - w1; + w2 = info.second.width - w1; } - Eigen::Matrix backup_1 = - _labels.block(info.second.t, p1, info.second.h, w1); - Eigen::Matrix backup_2 = - _labels.block(info.second.t, p2, info.second.h, w2); + Eigen::Matrix backup_1 = _labels.block(info.second.top, p1, info.second.height, w1); + Eigen::Matrix backup_2 = _labels.block(info.second.top, p2, info.second.height, w2); double base_cost = cost(info.first); alphaExpansion(info.first); @@ -352,8 +368,8 @@ class GraphcutSeams if(new_cost > base_cost) { - _labels.block(info.second.t, p1, info.second.h, w1) = backup_1; - _labels.block(info.second.t, p2, info.second.h, w2) = backup_2; + _labels.block(info.second.top, p1, info.second.height, w1) = backup_1; + _labels.block(info.second.top, p2, info.second.height, w2) = backup_2; } else if(new_cost < base_cost) { @@ -373,20 +389,20 @@ class GraphcutSeams double cost(IndexT currentLabel) { - Rect rect = _rects[currentLabel]; + BoundingBox rect = _rects[currentLabel]; double cost = 0.0; - for(int i = 0; i < rect.h - 1; i++) + for(int i = 0; i < rect.height - 1; i++) { - int y = rect.t + i; + int y = rect.top + i; int yp = y + 1; - for(int j = 0; j < rect.w; j++) + for(int j = 0; j < rect.width; j++) { - int x = rect.l + j; + int x = rect.left + j; if(x >= _owners.Width()) { x = x - _owners.Width(); @@ -520,26 +536,26 @@ class GraphcutSeams bool alphaExpansion(IndexT currentLabel) { - Rect rect = _rects[currentLabel]; + BoundingBox rect = _rects[currentLabel]; - image::Image mask(rect.w, rect.h, true, 0); - image::Image ids(rect.w, rect.h, true, -1); - image::Image color_label(rect.w, rect.h, true, image::RGBfColor(0.0f, 0.0f, 0.0f)); - image::Image color_other(rect.w, rect.h, true, image::RGBfColor(0.0f, 0.0f, 0.0f)); + image::Image mask(rect.width, rect.height, true, 0); + image::Image ids(rect.width, rect.height, true, -1); + image::Image color_label(rect.width, rect.height, true, image::RGBfColor(0.0f, 0.0f, 0.0f)); + image::Image color_other(rect.width, rect.height, true, image::RGBfColor(0.0f, 0.0f, 0.0f)); /*Compute distance map to seams*/ - image::Image distanceMap(rect.w, rect.h); + image::Image distanceMap(rect.width, rect.height); { - image::Image binarizedWorld(rect.w, rect.h); + image::Image binarizedWorld(rect.width, rect.height); - for(int i = 0; i < rect.h; i++) + for(int i = 0; i < rect.height; i++) { - int y = rect.t + i; + int y = rect.top + i; - for(int j = 0; j < rect.w; j++) + for(int j = 0; j < rect.width; j++) { - int x = rect.l + j; + int x = rect.left + j; if(x >= _owners.Width()) { x = x - _owners.Width(); @@ -557,7 +573,7 @@ class GraphcutSeams } } - image::Image seams(rect.w, rect.h); + image::Image seams(rect.width, rect.height); if(!computeSeamsMap(seams, binarizedWorld)) { return false; @@ -578,15 +594,15 @@ class GraphcutSeams - 2 if the pixel is viewed by *another* label and this label is marked as current valid label - 3 if the pixel is 1 + 2 : the pixel is not selected as alpha territory, but alpha is looking at it */ - for(int i = 0; i < rect.h; i++) + for(int i = 0; i < rect.height; i++) { - int y = rect.t + i; + int y = rect.top + i; - for(int j = 0; j < rect.w; j++) + for(int j = 0; j < rect.width; j++) { - int x = rect.l + j; + int x = rect.left + j; if(x >= _owners.Width()) { x = x - _owners.Width(); @@ -663,9 +679,9 @@ class GraphcutSeams Let's create an index per valid pixels for graph cut reference */ int count = 0; - for(int i = 0; i < rect.h; i++) + for(int i = 0; i < rect.height; i++) { - for(int j = 0; j < rect.w; j++) + for(int j = 0; j < rect.width; j++) { if(mask(i, j) == 0) { @@ -681,9 +697,9 @@ class GraphcutSeams MaxFlow_AdjList gc(count); size_t countValid = 0; - for(int i = 0; i < rect.h; i++) + for(int i = 0; i < rect.height; i++) { - for(int j = 0; j < rect.w; j++) + for(int j = 0; j < rect.width; j++) { /* If this pixel is not valid, ignore */ @@ -697,8 +713,8 @@ class GraphcutSeams int im1 = std::max(i - 1, 0); int jm1 = std::max(j - 1, 0); - int ip1 = std::min(i + 1, rect.h - 1); - int jp1 = std::min(j + 1, rect.w - 1); + int ip1 = std::min(i + 1, rect.height - 1); + int jp1 = std::min(j + 1, rect.width - 1); if(mask(i, j) == 1) { @@ -762,9 +778,9 @@ class GraphcutSeams When two neighboor pixels have different labels, there is a seam (border) cost. Graph cut will try to make sure the territory will have a minimal border cost */ - for(int i = 0; i < rect.h; i++) + for(int i = 0; i < rect.height; i++) { - for(int j = 0; j < rect.w; j++) + for(int j = 0; j < rect.width; j++) { if(mask(i, j) == 0) @@ -825,15 +841,15 @@ class GraphcutSeams gc.compute(); int changeCount = 0; - for(int i = 0; i < rect.h; i++) + for(int i = 0; i < rect.height; i++) { - int y = rect.t + i; + int y = rect.top + i; - for(int j = 0; j < rect.w; j++) + for(int j = 0; j < rect.width; j++) { - int x = rect.l + j; + int x = rect.left + j; if(x >= _owners.Width()) { x = x - _owners.Width(); @@ -853,20 +869,27 @@ class GraphcutSeams _labels(y, x) = currentLabel; } } - } + }*/ return true; } - const image::Image& getLabels() { return _labels; } + CachedImage & getLabels() + { + return _labels; + } private: - std::map _rects; - ImageOwners _owners; - image::Image _labels; - image::Image _original_labels; - image::Image _distancesSeams; + + std::map _rects; + int _outputWidth; + int _outputHeight; size_t _maximal_distance_change; + + CachedImage _labels; + CachedImage _original_labels; + CachedImage _distancesSeams; + ImageOwners _owners; }; } // namespace aliceVision \ No newline at end of file diff --git a/src/aliceVision/panorama/imageOps.cpp b/src/aliceVision/panorama/imageOps.cpp index 2667537afc..a8753794f2 100644 --- a/src/aliceVision/panorama/imageOps.cpp +++ b/src/aliceVision/panorama/imageOps.cpp @@ -3,34 +3,36 @@ namespace aliceVision { -void removeNegativeValues(aliceVision::image::Image& img) +void removeNegativeValues(CachedImage & img) { - for(int i = 0; i < img.Height(); i++) - { - for(int j = 0; j < img.Width(); j++) + img.perPixelOperation( + [](const image::RGBfColor & c) -> image::RGBfColor { - image::RGBfColor& pix = img(i, j); image::RGBfColor rpix; - rpix.r() = std::exp(pix.r()); - rpix.g() = std::exp(pix.g()); - rpix.b() = std::exp(pix.b()); + image::RGBfColor ret; + + rpix.r() = std::exp(c.r()); + rpix.g() = std::exp(c.g()); + rpix.b() = std::exp(c.b()); if(rpix.r() < 0.0) { - pix.r() = 0.0; + ret.r() = 0.0; } if(rpix.g() < 0.0) { - pix.g() = 0.0; + ret.g() = 0.0; } if(rpix.b() < 0.0) { - pix.b() = 0.0; + ret.b() = 0.0; } + + return ret; } - } + ); } } // namespace aliceVision \ No newline at end of file diff --git a/src/aliceVision/panorama/imageOps.hpp b/src/aliceVision/panorama/imageOps.hpp index d19472742f..69333c1897 100644 --- a/src/aliceVision/panorama/imageOps.hpp +++ b/src/aliceVision/panorama/imageOps.hpp @@ -1,6 +1,7 @@ #pragma once #include +#include "cachedImage.hpp" namespace aliceVision { @@ -111,6 +112,6 @@ bool addition(aliceVision::image::Image& AplusB, const aliceVision::image::Im return true; } -void removeNegativeValues(aliceVision::image::Image& img); +void removeNegativeValues(CachedImage& img); } // namespace aliceVision \ No newline at end of file diff --git a/src/aliceVision/panorama/laplacianCompositer.hpp b/src/aliceVision/panorama/laplacianCompositer.hpp index 5131b0dfc6..b067a9a971 100644 --- a/src/aliceVision/panorama/laplacianCompositer.hpp +++ b/src/aliceVision/panorama/laplacianCompositer.hpp @@ -1,13 +1,18 @@ #pragma once #include "compositer.hpp" +#include "feathering.hpp" +#include "laplacianPyramid.hpp" namespace aliceVision { template -bool makeImagePyramidCompatible(image::Image& output, size_t& out_offset_x, size_t& out_offset_y, - const image::Image& input, size_t offset_x, size_t offset_y, size_t num_levels) +bool makeImagePyramidCompatible(image::Image& output, + size_t& out_offset_x, size_t& out_offset_y, + const image::Image& input, + size_t offset_x, size_t offset_y, + size_t num_levels) { if(num_levels == 0) @@ -72,11 +77,21 @@ class LaplacianCompositer : public Compositer { } + virtual bool initialize(image::TileCacheManager::shared_ptr & cacheManager) + { + if (!Compositer::initialize(cacheManager)) + { + return false; + } + + return _pyramid_panorama.initialize(cacheManager); + } + virtual bool append(const aliceVision::image::Image& color, const aliceVision::image::Image& inputMask, const aliceVision::image::Image& inputWeights, size_t offset_x, size_t offset_y) { - /*Get smalles size*/ + /*Get smallest size*/ size_t minsize = std::min(color.Height(), color.Width()); /* @@ -94,34 +109,40 @@ class LaplacianCompositer : public Compositer return false; } + //If the input scale is more important than previously processed, + // The pyramid must be deepened accordingly if(optimal_scale > _bands) { - _bands = optimal_scale; - _pyramid_panorama.augment(_bands); + //_bands = optimal_scale; + //_pyramid_panorama.augment(_bands); } + // Make sure input is compatible with pyramid processing size_t new_offset_x, new_offset_y; aliceVision::image::Image color_pot; aliceVision::image::Image mask_pot; aliceVision::image::Image weights_pot; + makeImagePyramidCompatible(color_pot, new_offset_x, new_offset_y, color, offset_x, offset_y, _bands); makeImagePyramidCompatible(mask_pot, new_offset_x, new_offset_y, inputMask, offset_x, offset_y, _bands); makeImagePyramidCompatible(weights_pot, new_offset_x, new_offset_y, inputWeights, offset_x, offset_y, _bands); + //std::cout << new_offset_x << " " << new_offset_y << std::endl; + + // Fill Color images masked parts with fake but coherent info aliceVision::image::Image feathered; feathering(feathered, color_pot, mask_pot); /*To log space for hdr*/ - for(int i = 0; i < feathered.Height(); i++) + /*for(int i = 0; i < feathered.Height(); i++) { for(int j = 0; j < feathered.Width(); j++) { - feathered(i, j).r() = std::log(std::max(1e-8f, feathered(i, j).r())); feathered(i, j).g() = std::log(std::max(1e-8f, feathered(i, j).g())); feathered(i, j).b() = std::log(std::max(1e-8f, feathered(i, j).b())); } - } + }*/ _pyramid_panorama.apply(feathered, mask_pot, weights_pot, new_offset_x, new_offset_y); @@ -133,16 +154,18 @@ class LaplacianCompositer : public Compositer _pyramid_panorama.rebuild(_panorama); - /*Go back to normal space from log space*/ - for(int i = 0; i < _panorama.Height(); i++) - { - for(int j = 0; j < _panorama.Width(); j++) - { - _panorama(i, j).r() = std::exp(_panorama(i, j).r()); - _panorama(i, j).g() = std::exp(_panorama(i, j).g()); - _panorama(i, j).b() = std::exp(_panorama(i, j).b()); + _panorama.perPixelOperation( + [](const image::RGBAfColor & a) -> image::RGBAfColor { + + image::RGBAfColor out; + + out.r() = std::exp(a.r()); + out.g() = std::exp(a.g()); + out.b() = std::exp(a.b()); + + return out; } - } + ); return true; } diff --git a/src/aliceVision/panorama/laplacianPyramid.cpp b/src/aliceVision/panorama/laplacianPyramid.cpp index 861a0aa0fc..2e92b2cc09 100644 --- a/src/aliceVision/panorama/laplacianPyramid.cpp +++ b/src/aliceVision/panorama/laplacianPyramid.cpp @@ -2,42 +2,82 @@ #include "feathering.hpp" #include "gaussian.hpp" +#include "compositer.hpp" namespace aliceVision { -LaplacianPyramid::LaplacianPyramid(size_t base_width, size_t base_height, size_t max_levels) +LaplacianPyramid::LaplacianPyramid(size_t base_width, size_t base_height, size_t max_levels) : +_baseWidth(base_width), +_baseHeight(base_height), +_maxLevels(max_levels) { +} - size_t width = base_width; - size_t height = base_height; +bool LaplacianPyramid::initialize(image::TileCacheManager::shared_ptr & cacheManager) +{ + size_t width = _baseWidth; + size_t height = _baseHeight; /*Make sure pyramid size can be divided by 2 on each levels*/ - double max_scale = 1.0 / pow(2.0, max_levels - 1); - // width = size_t(ceil(double(width) * max_scale) / max_scale); - // height = size_t(ceil(double(height) * max_scale) / max_scale); + double max_scale = 1.0 / pow(2.0, _maxLevels - 1); /*Prepare pyramid*/ - for(int lvl = 0; lvl < max_levels; lvl++) + for(int lvl = 0; lvl < _maxLevels; lvl++) { + CachedImage color; + CachedImage weights; + + if(!color.createImage(cacheManager, width, height)) + { + return false; + } + + if(!weights.createImage(cacheManager, width, height)) + { + return false; + } + + if(!color.perPixelOperation( + [](image::RGBfColor ) -> image::RGBfColor + { + return image::RGBfColor(0.0f, 0.0f, 0.0f); + }) + ) + { + return false; + } - _levels.push_back( - aliceVision::image::Image(width, height, true, image::RGBfColor(0.0f, 0.0f, 0.0f))); - _weights.push_back(aliceVision::image::Image(width, height, true, 0.0f)); + if(!weights.perPixelOperation( + [](float) -> float + { + return 0.0f; + }) + ) + { + return false; + } + + _levels.push_back(color); + _weights.push_back(weights); height /= 2; width /= 2; } + + return true; } bool LaplacianPyramid::augment(size_t new_max_levels) { - if(new_max_levels <= _levels.size()) + /*if(new_max_levels <= _levels.size()) { return false; } + _maxLevels = new_max_levels; + size_t old_max_level = _levels.size(); image::Image current_color = _levels[_levels.size() - 1]; @@ -118,7 +158,7 @@ bool LaplacianPyramid::augment(size_t new_max_levels) height /= 2; } - merge(current_color, current_weights, _levels.size() - 1, 0, 0); + merge(current_color, current_weights, _levels.size() - 1, 0, 0);*/ return true; } @@ -128,6 +168,8 @@ bool LaplacianPyramid::apply(const aliceVision::image::Image& const aliceVision::image::Image& weights, size_t offset_x, size_t offset_y) { + + int width = source.Width(); int height = source.Height(); @@ -251,68 +293,88 @@ bool LaplacianPyramid::apply(const aliceVision::image::Image& return true; } + bool LaplacianPyramid::merge(const aliceVision::image::Image& oimg, const aliceVision::image::Image& oweight, size_t level, size_t offset_x, size_t offset_y) { + CachedImage & img = _levels[level]; + CachedImage & weight = _weights[level]; - image::Image& img = _levels[level]; - image::Image& weight = _weights[level]; + aliceVision::image::Image extractedColor(oimg.Width(), oimg.Height()); + aliceVision::image::Image extractedWeight(oimg.Width(), oimg.Height()); - for(int i = 0; i < oimg.Height(); i++) - { - int di = i + offset_y; - if(di >= img.Height()) - continue; + BoundingBox extractBb; + extractBb.left = offset_x; + extractBb.top = offset_y; + extractBb.width = oimg.Width(); + extractBb.height = oimg.Height(); + + if (!loopyCachedImageExtract(extractedColor, img, extractBb)) + { + return false; + } + + if (!loopyCachedImageExtract(extractedWeight, weight, extractBb)) + { + return false; + } + + for(int i = 0; i < oimg.Height(); i++) + { for(int j = 0; j < oimg.Width(); j++) { + extractedColor(i, j).r() += oimg(i, j).r() * oweight(i, j); + extractedColor(i, j).g() += oimg(i, j).g() * oweight(i, j); + extractedColor(i, j).b() += oimg(i, j).b() * oweight(i, j); + extractedWeight(i, j) += oweight(i, j); + } + } - int dj = j + offset_x; - if(dj >= weight.Width()) - { - dj = dj - weight.Width(); - } + if (!loopyCachedImageAssign(img, extractedColor, extractBb)) { + return false; + } - img(di, dj).r() += oimg(i, j).r() * oweight(i, j); - img(di, dj).g() += oimg(i, j).g() * oweight(i, j); - img(di, dj).b() += oimg(i, j).b() * oweight(i, j); - weight(di, dj) += oweight(i, j); - } + if (!loopyCachedImageAssign(weight, extractedWeight, extractBb)) { + return false; } return true; } -bool LaplacianPyramid::rebuild(image::Image& output) +bool LaplacianPyramid::rebuild(CachedImage& output) { for(int l = 0; l < _levels.size(); l++) { - for(int i = 0; i < _levels[l].Height(); i++) - { - for(int j = 0; j < _levels[l].Width(); j++) + _levels[l].perPixelOperation(_weights[l], + [](const image::RGBfColor & c, const float & w) -> image::RGBfColor { - if(_weights[l](i, j) < 1e-6) + if (w < 1e-6) { - _levels[l](i, j) = image::RGBfColor(0.0); - continue; + return image::RGBfColor(0.0f, 0.0f, 0.0f); } - _levels[l](i, j).r() = _levels[l](i, j).r() / _weights[l](i, j); - _levels[l](i, j).g() = _levels[l](i, j).g() / _weights[l](i, j); - _levels[l](i, j).b() = _levels[l](i, j).b() / _weights[l](i, j); + image::RGBfColor r; + + r.r() = c.r() / w; + r.g() = c.g() / w; + r.b() = c.b() / w; + + return r; } - } + ); } + removeNegativeValues(_levels[_levels.size() - 1]); for(int l = _levels.size() - 2; l >= 0; l--) { - aliceVision::image::Image buf(_levels[l].Width(), _levels[l].Height()); + /*aliceVision::image::Image buf(_levels[l].Width(), _levels[l].Height()); aliceVision::image::Image buf2(_levels[l].Width(), _levels[l].Height()); upscale(buf, _levels[l + 1]); @@ -326,12 +388,12 @@ bool LaplacianPyramid::rebuild(image::Image& output) } } - addition(_levels[l], _levels[l], buf2); + addition(_levels[l], _levels[l], buf2);*/ removeNegativeValues(_levels[l]); } // Write output to RGBA - for(int i = 0; i < output.Height(); i++) + /*for(int i = 0; i < output.Height(); i++) { for(int j = 0; j < output.Width(); j++) { @@ -348,7 +410,7 @@ bool LaplacianPyramid::rebuild(image::Image& output) output(i, j).a() = 1.0f; } } - } + }*/ return true; } diff --git a/src/aliceVision/panorama/laplacianPyramid.hpp b/src/aliceVision/panorama/laplacianPyramid.hpp index 87b86fafdd..804ef0d53e 100644 --- a/src/aliceVision/panorama/laplacianPyramid.hpp +++ b/src/aliceVision/panorama/laplacianPyramid.hpp @@ -2,6 +2,8 @@ #include "imageOps.hpp" +#include "cachedImage.hpp" + namespace aliceVision { @@ -9,17 +11,27 @@ class LaplacianPyramid { public: LaplacianPyramid(size_t base_width, size_t base_height, size_t max_levels); + + bool initialize(image::TileCacheManager::shared_ptr & cacheManager); + bool augment(size_t new_max_levels); + bool apply(const aliceVision::image::Image& source, const aliceVision::image::Image& mask, const aliceVision::image::Image& weights, size_t offset_x, size_t offset_y); + bool merge(const aliceVision::image::Image& oimg, const aliceVision::image::Image& oweight, size_t level, size_t offset_x, size_t offset_y); - bool rebuild(image::Image& output); + + bool rebuild(CachedImage& output); private: - std::vector> _levels; - std::vector> _weights; + int _baseWidth; + int _baseHeight; + int _maxLevels; + + std::vector> _levels; + std::vector> _weights; }; } // namespace aliceVision \ No newline at end of file diff --git a/src/aliceVision/panorama/seams.cpp b/src/aliceVision/panorama/seams.cpp index 91ba2bd578..ae70ab6883 100644 --- a/src/aliceVision/panorama/seams.cpp +++ b/src/aliceVision/panorama/seams.cpp @@ -226,13 +226,13 @@ bool WTASeams::append(const aliceVision::image::Image& inputMask, return true; } -void HierarchicalGraphcutSeams::setOriginalLabels(const image::Image& labels) +void HierarchicalGraphcutSeams::setOriginalLabels(CachedImage& labels) { /* First of all, Propagate label to all levels */ - image::Image current_label = labels; + /*image::Image current_label = labels; for(int l = 1; l <= _levelOfInterest; l++) { @@ -254,14 +254,14 @@ void HierarchicalGraphcutSeams::setOriginalLabels(const image::Image& la current_label = next_label; } - _graphcut->setOriginalLabels(current_label); + _graphcut->setOriginalLabels(current_label);*/ } bool HierarchicalGraphcutSeams::append(const aliceVision::image::Image& input, const aliceVision::image::Image& inputMask, IndexT currentIndex, size_t offset_x, size_t offset_y) { - image::Image current_color = input; + /*image::Image current_color = input; image::Image current_mask = inputMask; for(int l = 1; l <= _levelOfInterest; l++) @@ -300,13 +300,15 @@ bool HierarchicalGraphcutSeams::append(const aliceVision::image::Imageappend(current_color, current_mask, currentIndex, offset_x, offset_y); + return _graphcut->append(current_color, current_mask, currentIndex, offset_x, offset_y);*/ + + return true; } bool HierarchicalGraphcutSeams::process() { - if(!_graphcut->process()) + /*if(!_graphcut->process()) { return false; } @@ -340,33 +342,47 @@ bool HierarchicalGraphcutSeams::process() current_labels = next_label; } - _labels = current_labels; + _labels = current_labels;*/ return true; } -void getMaskFromLabels(aliceVision::image::Image & mask, aliceVision::image::Image & labels, IndexT index, size_t offset_x, size_t offset_y) { - - for (int i = 0; i < mask.Height(); i++) { +bool HierarchicalGraphcutSeams::initialize(image::TileCacheManager::shared_ptr & cacheManager) +{ + return true; +} - int di = i + offset_y; +bool getMaskFromLabels(aliceVision::image::Image & mask, CachedImage & labels, IndexT index, size_t offset_x, size_t offset_y) { - for (int j = 0; j < mask.Width(); j++) { + image::Image extractedLabels(mask.Width(), mask.Height()); - int dj = j + offset_x; - if (dj >= labels.Width()) { - dj = dj - labels.Width(); - } + BoundingBox bb; + bb.left = offset_x; + bb.top = offset_y; + bb.width = mask.Width(); + bb.height = mask.Height(); + if (!loopyCachedImageExtract(extractedLabels, labels, bb)) + { + return false; + } - if (labels(di, dj) == index) { - mask(i, j) = 1.0f; - } - else { - mask(i, j) = 0.0f; - } + for (int i = 0; i < extractedLabels.Height(); i++) + { + for (int j = 0; j < extractedLabels.Width(); j++) + { + if (extractedLabels(i, j) == index) + { + mask(i, j) = 1.0f; + } + else + { + mask(i, j) = 0.0f; + } + } } - } + + return true; } } // namespace aliceVision \ No newline at end of file diff --git a/src/aliceVision/panorama/seams.hpp b/src/aliceVision/panorama/seams.hpp index 22e4795748..e5e762134e 100644 --- a/src/aliceVision/panorama/seams.hpp +++ b/src/aliceVision/panorama/seams.hpp @@ -4,7 +4,7 @@ #include #include "cachedImage.hpp" -#include "graphcut.hpp" +//#include "graphcut.hpp" namespace aliceVision { @@ -13,7 +13,7 @@ void drawBorders(aliceVision::image::Image& inout, aliceVisio size_t offset_x, size_t offset_y); void drawSeams(aliceVision::image::Image& inout, aliceVision::image::Image& labels); -void getMaskFromLabels(aliceVision::image::Image & mask, aliceVision::image::Image & labels, IndexT index, size_t offset_x, size_t offset_y); +bool getMaskFromLabels(aliceVision::image::Image & mask, CachedImage & labels, IndexT index, size_t offset_x, size_t offset_y); class WTASeams { @@ -52,21 +52,19 @@ class HierarchicalGraphcutSeams : _outputWidth(outputWidth) , _outputHeight(outputHeight) , _levelOfInterest(levelOfInterest) - , _labels(outputWidth, outputHeight, true, UndefinedIndexT) { - - double scale = 1.0 / pow(2.0, levelOfInterest); - size_t width = size_t(floor(double(outputWidth) * scale)); - size_t height = size_t(floor(double(outputHeight) * scale)); - - _graphcut = std::unique_ptr(new GraphcutSeams(width, height)); } virtual ~HierarchicalGraphcutSeams() = default; - void setOriginalLabels(const image::Image& labels); + bool initialize(image::TileCacheManager::shared_ptr & cacheManager); + + void setOriginalLabels(CachedImage& labels); - void setMaximalDistance(int distance) { _graphcut->setMaximalDistance(distance); } + void setMaximalDistance(int distance) + { + //_graphcut->setMaximalDistance(distance); + } virtual bool append(const aliceVision::image::Image& input, const aliceVision::image::Image& inputMask, IndexT currentIndex, size_t offset_x, @@ -74,11 +72,14 @@ class HierarchicalGraphcutSeams bool process(); - const image::Image& getLabels() { return _labels; } + CachedImage& getLabels() + { + return _labels; + } private: - std::unique_ptr _graphcut; - image::Image _labels; + //std::unique_ptr _graphcut; + CachedImage _labels; size_t _levelOfInterest; size_t _outputWidth; size_t _outputHeight; diff --git a/src/software/pipeline/main_panoramaCompositing.cpp b/src/software/pipeline/main_panoramaCompositing.cpp index b94812ed00..31624ce597 100644 --- a/src/software/pipeline/main_panoramaCompositing.cpp +++ b/src/software/pipeline/main_panoramaCompositing.cpp @@ -30,6 +30,7 @@ #include #include #include +#include #include // These constants define the current software version. @@ -85,6 +86,72 @@ bool computeWTALabels(CachedImage & labels, image::TileCacheManager::sha return true; } +bool computeGCLabels(CachedImage & labels, image::TileCacheManager::shared_ptr& cacheManager, const sfmData::SfMData& sfmData, const std::string & inputPath, std::pair & panoramaSize) +{ + + //Compute finest level possible for graph cut + int initial_level = 0; + int max_width_for_graphcut = 5000; + double ratio = double(panoramaSize.first) / double(max_width_for_graphcut); + if (ratio > 1.0) { + initial_level = int(ceil(log2(ratio))); + } + + for (int l = initial_level; l>= 0; l--) + { + HierarchicalGraphcutSeams seams(panoramaSize.first, panoramaSize.second, l); + + if (!seams.initialize(cacheManager)) + { + return false; + } + + seams.setOriginalLabels(labels); + + if (l != initial_level) + { + seams.setMaximalDistance(100); + } + + for (const auto& viewIt : sfmData.getViews()) + { + if(!sfmData.isPoseAndIntrinsicDefined(viewIt.second.get())) + { + // skip unreconstructed views + continue; + } + + // Load mask + const std::string maskPath = (fs::path(inputPath) / (std::to_string(viewIt.first) + "_mask.exr")).string(); + ALICEVISION_LOG_INFO("Load mask with path " << maskPath); + image::Image mask; + image::readImage(maskPath, mask, image::EImageColorSpace::NO_CONVERSION); + + // Get offset + oiio::ParamValueList metadata = image::readImageMetadata(maskPath); + const std::size_t offsetX = metadata.find("AliceVision:offsetX")->get_int(); + const std::size_t offsetY = metadata.find("AliceVision:offsetY")->get_int(); + + // Load Color + const std::string colorsPath = (fs::path(inputPath) / (std::to_string(viewIt.first) + ".exr")).string(); + ALICEVISION_LOG_INFO("Load colors with path " << colorsPath); + image::Image colors; + image::readImage(colorsPath, colors, image::EImageColorSpace::NO_CONVERSION); + + // Append to graph cut + seams.append(colors, mask, viewIt.first, offsetX, offsetY); + } + + if (seams.process()) + { + labels = seams.getLabels(); + } + } + + + return true; +} + int aliceVision_main(int argc, char** argv) { std::string sfmDataFilepath; @@ -211,13 +278,24 @@ int aliceVision_main(int argc, char** argv) // Configure the cache manager memory cacheManager->setInCoreMaxObjectCount(1000); - AlphaCompositer compositer(panoramaSize.first, panoramaSize.second); + LaplacianCompositer compositer(panoramaSize.first, panoramaSize.second, 3); compositer.initialize(cacheManager); CachedImage labels; - computeWTALabels(labels, cacheManager, sfmData, warpingFolder, panoramaSize); + if (!computeWTALabels(labels, cacheManager, sfmData, warpingFolder, panoramaSize)) + { + ALICEVISION_LOG_ERROR("Error computing initial labels"); + return EXIT_FAILURE; + } + + /*if (!computeGCLabels(labels, cacheManager, sfmData, warpingFolder, panoramaSize)) + { + ALICEVISION_LOG_ERROR("Error computing graph cut labels"); + return EXIT_FAILURE; + }*/ + - /*for(const auto& it : sfmData.getViews()) + for(const auto& it : sfmData.getViews()) { IndexT viewId = it.first; auto view = it.second; @@ -230,7 +308,7 @@ int aliceVision_main(int argc, char** argv) // Load image and convert it to linear colorspace const std::string imagePath = (fs::path(warpingFolder) / (std::to_string(viewId) + ".exr")).string(); - ALICEVISION_LOG_INFO("Load image with path " << imagePath); + //ALICEVISION_LOG_INFO("Load image with path " << imagePath); image::Image source; image::readImage(imagePath, source, image::EImageColorSpace::NO_CONVERSION); @@ -240,22 +318,30 @@ int aliceVision_main(int argc, char** argv) // Load mask const std::string maskPath = (fs::path(warpingFolder) / (std::to_string(viewId) + "_mask.exr")).string(); - ALICEVISION_LOG_INFO("Load mask with path " << maskPath); + //ALICEVISION_LOG_INFO("Load mask with path " << maskPath); image::Image mask; image::readImage(maskPath, mask, image::EImageColorSpace::NO_CONVERSION); // Load Weights - const std::string weightsPath = (fs::path(warpingFolder) / (std::to_string(viewId) + "_weight.exr")).string(); + /*const std::string weightsPath = (fs::path(warpingFolder) / (std::to_string(viewId) + "_weight.exr")).string(); ALICEVISION_LOG_INFO("Load weights with path " << weightsPath); image::Image weights; - image::readImage(weightsPath, weights, image::EImageColorSpace::NO_CONVERSION); + image::readImage(weightsPath, weights, image::EImageColorSpace::NO_CONVERSION);*/ - compositer.append(source, mask, weights, offsetX, offsetY); - }*/ + + image::Image seams(mask.Width(), mask.Height()); + if (!getMaskFromLabels(seams, labels, viewId, offsetX, offsetY)) + { + ALICEVISION_LOG_ERROR("Error estimating seams image"); + return EXIT_FAILURE; + } + + compositer.append(source, mask, seams, offsetX, offsetY); + } compositer.terminate(); - //compositer.save(outputPanorama); + compositer.save(outputPanorama); return EXIT_SUCCESS; } From 2557ba08c65d4a9b30be6013e5ec014c24c40b69 Mon Sep 17 00:00:00 2001 From: Fabien Servant Date: Mon, 5 Oct 2020 10:03:18 +0200 Subject: [PATCH 06/79] [panorama] accelerate warping --- .../pipeline/main_panoramaWarping.cpp | 159 +++++++++--------- 1 file changed, 84 insertions(+), 75 deletions(-) diff --git a/src/software/pipeline/main_panoramaWarping.cpp b/src/software/pipeline/main_panoramaWarping.cpp index e8e0cee06e..05a3ac6a8e 100644 --- a/src/software/pipeline/main_panoramaWarping.cpp +++ b/src/software/pipeline/main_panoramaWarping.cpp @@ -280,36 +280,41 @@ int aliceVision_main(int argc, char** argv) //Initialize bouding box for image BoundingBox globalBbox; + + std::vector boxes; + for (int y = 0; y < snappedCoarseBbox.height; y += tileSize) + { + for (int x = 0; x < snappedCoarseBbox.width; x += tileSize) + { + BoundingBox localBbox; + localBbox.left = x + snappedCoarseBbox.left; + localBbox.top = y + snappedCoarseBbox.top; + localBbox.width = tileSize; + localBbox.height = tileSize; + boxes.push_back(localBbox); + } + } #pragma omp parallel for - for (int y = 0; y < snappedCoarseBbox.height; y += tileSize) { + for (int i = 0; i < boxes.size(); i++) { - for (int x = 0; x < snappedCoarseBbox.width; x += tileSize) { + BoundingBox localBbox = boxes[i]; - BoundingBox localBbox; - localBbox.left = x + snappedCoarseBbox.left; - localBbox.top = y + snappedCoarseBbox.top; - localBbox.width = tileSize; - localBbox.height = tileSize; + /*int stillToProcess = coarseBbox.width - localBbox.left; + if (stillToProcess < tileSize) { + localBbox.width = stillToProcess; + }*/ - - - /*int stillToProcess = coarseBbox.width - localBbox.left; - if (stillToProcess < tileSize) { - localBbox.width = stillToProcess; - }*/ - - // Prepare coordinates map - CoordinatesMap map; - if (!map.build(panoramaSize, camPose, *(intrinsic.get()), localBbox)) { - continue; - } + // Prepare coordinates map + CoordinatesMap map; + if (!map.build(panoramaSize, camPose, *(intrinsic.get()), localBbox)) { + continue; + } - #pragma omp critical - { - globalBbox.unionWith(map.getBoundingBox()); - } - } + #pragma omp critical + { + globalBbox.unionWith(map.getBoundingBox()); + } } @@ -383,61 +388,65 @@ int aliceVision_main(int argc, char** argv) continue; } + boxes.clear(); + for (int y = 0; y < snappedGlobalBbox.height; y += tileSize) + { + for (int x = 0; x < snappedGlobalBbox.width; x += tileSize) + { + BoundingBox localBbox; + localBbox.left = x + snappedGlobalBbox.left; + localBbox.top = y + snappedGlobalBbox.top; + localBbox.width = tileSize; + localBbox.height = tileSize; + boxes.push_back(localBbox); + } + } + + #pragma omp parallel for { - for (int y = 0; y < snappedGlobalBbox.height; y += tileSize) + for (int i = 0; i < boxes.size(); i++) { - for (int x = 0; x < snappedGlobalBbox.width; x += tileSize) + BoundingBox localBbox = boxes[i]; + + int x = localBbox.left - snappedGlobalBbox.left; + int y = localBbox.top - snappedGlobalBbox.top; + + // Prepare coordinates map + CoordinatesMap map; + if (!map.build(panoramaSize, camPose, *(intrinsic.get()), localBbox)) + { + continue; + } + + // Warp image + GaussianWarper warper; + if (!warper.warp(map, pyramid)) { + continue; + } + + // Alpha mask + aliceVision::image::Image weights; + if (!distanceToCenter(weights, map, intrinsic->w(), intrinsic->h())) { + continue; + } + + // Store + #pragma omp critical + { + out_view->write_tile(x, y, 0, oiio::TypeDesc::FLOAT, warper.getColor().data()); + } + + // Store + #pragma omp critical + { + out_mask->write_tile(x, y, 0, oiio::TypeDesc::UCHAR, warper.getMask().data()); + } + + // Store + #pragma omp critical { - BoundingBox localBbox; - localBbox.left = x + snappedGlobalBbox.left; - localBbox.top = y + snappedGlobalBbox.top; - localBbox.width = tileSize; - localBbox.height = tileSize; - - /*bool fillup = false; - int stillToProcess = globalBbox.width - localBbox.left; - if (stillToProcess < tileSize) { - to continue - fillup = true; - }*/ - - // Prepare coordinates map - CoordinatesMap map; - if (!map.build(panoramaSize, camPose, *(intrinsic.get()), localBbox)) - { - continue; - } - - // Warp image - GaussianWarper warper; - if (!warper.warp(map, pyramid)) { - continue; - } - - // Alpha mask - aliceVision::image::Image weights; - if (!distanceToCenter(weights, map, intrinsic->w(), intrinsic->h())) { - continue; - } - - // Store - #pragma omp critical - { - out_view->write_tile(x, y, 0, oiio::TypeDesc::FLOAT, warper.getColor().data()); - } - - // Store - #pragma omp critical - { - out_mask->write_tile(x, y, 0, oiio::TypeDesc::UCHAR, warper.getMask().data()); - } - - // Store - #pragma omp critical - { - out_weights->write_tile(x, y, 0, oiio::TypeDesc::FLOAT, weights.data()); - } + out_weights->write_tile(x, y, 0, oiio::TypeDesc::FLOAT, weights.data()); } } } From b62357e400dbd5723efd1c9c5c56479065468f14 Mon Sep 17 00:00:00 2001 From: Fabien Servant Date: Tue, 6 Oct 2020 08:10:52 +0200 Subject: [PATCH 07/79] [panorama] wip compositing --- src/aliceVision/panorama/alphaCompositer.hpp | 15 +- src/aliceVision/panorama/boundingBox.hpp | 65 +++- src/aliceVision/panorama/cachedImage.cpp | 43 +++ src/aliceVision/panorama/cachedImage.hpp | 21 ++ src/aliceVision/panorama/compositer.hpp | 49 ++- src/aliceVision/panorama/imageOps.cpp | 10 +- .../panorama/laplacianCompositer.hpp | 50 +-- src/aliceVision/panorama/laplacianPyramid.cpp | 295 ++++++++++++------ src/aliceVision/panorama/laplacianPyramid.hpp | 2 +- src/aliceVision/panorama/remapBbox.cpp | 3 +- src/aliceVision/panorama/seams.cpp | 10 +- .../pipeline/main_panoramaCompositing.cpp | 51 ++- 12 files changed, 454 insertions(+), 160 deletions(-) diff --git a/src/aliceVision/panorama/alphaCompositer.hpp b/src/aliceVision/panorama/alphaCompositer.hpp index fde368c1f1..c895c05eec 100644 --- a/src/aliceVision/panorama/alphaCompositer.hpp +++ b/src/aliceVision/panorama/alphaCompositer.hpp @@ -8,14 +8,15 @@ namespace aliceVision class AlphaCompositer : public Compositer { public: - AlphaCompositer(size_t outputWidth, size_t outputHeight) - : Compositer(outputWidth, outputHeight) + AlphaCompositer(image::TileCacheManager::shared_ptr & cacheManager, size_t outputWidth, size_t outputHeight) + : Compositer(cacheManager, outputWidth, outputHeight) { } virtual bool append(const aliceVision::image::Image& color, const aliceVision::image::Image& inputMask, - const aliceVision::image::Image& inputWeights, int offset_x, int offset_y) + const aliceVision::image::Image& inputWeights, + int offset_x, int offset_y, const BoundingBox & contentBox) { aliceVision::image::Image masked(color.Width(), color.Height()); @@ -50,7 +51,13 @@ class AlphaCompositer : public Compositer } } - if (!loopyCachedImageAssign(_panorama, masked, panoramaBb)) { + BoundingBox inputBb; + inputBb.left = 0; + inputBb.top = 0; + inputBb.width = masked.Width(); + inputBb.height = masked.Height(); + + if (!loopyCachedImageAssign(_panorama, masked, panoramaBb, inputBb)) { return false; } diff --git a/src/aliceVision/panorama/boundingBox.hpp b/src/aliceVision/panorama/boundingBox.hpp index 0a14252227..b542e89e5a 100644 --- a/src/aliceVision/panorama/boundingBox.hpp +++ b/src/aliceVision/panorama/boundingBox.hpp @@ -72,7 +72,8 @@ struct BoundingBox height = maxb - top + 1; } - bool isInside(const BoundingBox& other) const { + bool isInside(const BoundingBox& other) const + { if (other.left > left) return false; if (other.top > top) return false; @@ -82,4 +83,64 @@ struct BoundingBox return true; } -}; \ No newline at end of file + + BoundingBox dilate(int units) + { + BoundingBox b; + + b.left = left - units; + b.top = top - units; + b.width = width + units * 2; + b.height = height + units * 2; + + return b; + } + + void clampLeft() + { + if (left < 0) + { + width += left; + left = 0; + } + } + + void clampRight(int maxRight) + { + if (getRight() > maxRight) + { + int removal = getRight() - maxRight; + width -= removal; + } + } + + void clampTop() + { + if (top < 0) + { + height += top; + top = 0; + } + } + + void clampBottom(int maxBottom) + { + if (getBottom() > maxBottom) + { + int removal = getBottom() - maxBottom; + height -= removal; + } + } + + BoundingBox doubleSize() + { + BoundingBox b; + + b.left = left * 2; + b.top = top * 2; + b.width = width * 2; + b.height = height * 2; + + return b; + } +}; diff --git a/src/aliceVision/panorama/cachedImage.cpp b/src/aliceVision/panorama/cachedImage.cpp index d36e50177c..83b7803214 100644 --- a/src/aliceVision/panorama/cachedImage.cpp +++ b/src/aliceVision/panorama/cachedImage.cpp @@ -132,4 +132,47 @@ bool CachedImage::writeImage(const std::string& path) return true; } +template <> +bool CachedImage::writeImage(const std::string& path) +{ + + std::unique_ptr out = oiio::ImageOutput::create(path); + if(!out) + { + return false; + } + + oiio::ImageSpec spec(_memoryWidth, _memoryHeight, 1, oiio::TypeDesc::UINT8); + spec.tile_width = _tileSize; + spec.tile_height = _tileSize; + + if(!out->open(path, spec)) + { + return false; + } + + for(int i = 0; i < _tilesArray.size(); i++) + { + + std::vector& row = _tilesArray[i]; + + for(int j = 0; j < row.size(); j++) + { + + if(!row[j]->acquire()) + { + return false; + } + + unsigned char* ptr = row[j]->getDataPointer(); + + out->write_tile(j * _tileSize, i * _tileSize, 0, oiio::TypeDesc::UINT8, ptr); + } + } + + out->close(); + + return true; +} + } // namespace aliceVision \ No newline at end of file diff --git a/src/aliceVision/panorama/cachedImage.hpp b/src/aliceVision/panorama/cachedImage.hpp index 8da1fb2193..47e1e162a6 100644 --- a/src/aliceVision/panorama/cachedImage.hpp +++ b/src/aliceVision/panorama/cachedImage.hpp @@ -376,12 +376,29 @@ class CachedImage return true; } + bool fill(const T & val) + { + if (!perPixelOperation( + [val](T) -> T + { + return val; + }) + ) + { + return false; + } + + return true; + } + std::vector>& getTiles() { return _tilesArray; } int getWidth() const { return _width; } int getHeight() const { return _height; } + int getTileSize() const { return _tileSize; } + private: int _width; int _height; @@ -401,4 +418,8 @@ bool CachedImage::writeImage(const std::string& path); template <> bool CachedImage::writeImage(const std::string& path); + +template <> +bool CachedImage::writeImage(const std::string& path); + } // namespace aliceVision \ No newline at end of file diff --git a/src/aliceVision/panorama/compositer.hpp b/src/aliceVision/panorama/compositer.hpp index 5c62185d78..d41cc06336 100644 --- a/src/aliceVision/panorama/compositer.hpp +++ b/src/aliceVision/panorama/compositer.hpp @@ -8,16 +8,20 @@ namespace aliceVision { template -bool loopyCachedImageAssign(CachedImage & output, const aliceVision::image::Image & input, const BoundingBox & assignedOutputBb) +bool loopyCachedImageAssign(CachedImage & output, const aliceVision::image::Image & input, const BoundingBox & assignedOutputBb, const BoundingBox & assignedInputBb) { - BoundingBox inputBb; - BoundingBox outputBb; + BoundingBox inputBb = assignedInputBb; + BoundingBox outputBb = assignedOutputBb; + + if (inputBb.width != outputBb.width) + { + return false; + } - inputBb.left = 0; - inputBb.top = 0; - inputBb.width = input.Width(); - inputBb.height = input.Height(); - outputBb = assignedOutputBb; + if (inputBb.height != outputBb.height) + { + return false; + } if (outputBb.getBottom() >= output.getHeight()) { @@ -51,8 +55,13 @@ bool loopyCachedImageAssign(CachedImage & output, const aliceVision::image::I inputBb.left = width1; outputBb.left = 0; - inputBb.width = width2; - outputBb.width = width2; + + //no overlap + int width2_clamped = std::min(width2, left_1); + inputBb.width = width2_clamped; + outputBb.width = width2_clamped; + if (width2_clamped == 0) return true; + if (!output.assign(input, inputBb, outputBb)) { @@ -122,7 +131,8 @@ bool loopyCachedImageExtract(aliceVision::image::Image & output, CachedImage< class Compositer { public: - Compositer(int width, int height) : + Compositer(image::TileCacheManager::shared_ptr & cacheManager, int width, int height) : + _cacheManager(cacheManager), _panoramaWidth(width), _panoramaHeight(height) { @@ -131,7 +141,8 @@ class Compositer virtual bool append(const aliceVision::image::Image& color, const aliceVision::image::Image& inputMask, - const aliceVision::image::Image& inputWeights, int offset_x, int offset_y) + const aliceVision::image::Image& inputWeights, + int offset_x, int offset_y, const BoundingBox & contentBox) { aliceVision::image::Image masked(color.Width(), color.Height()); @@ -163,16 +174,22 @@ class Compositer } } - if (!loopyCachedImageAssign(_panorama, masked, panoramaBb)) { + BoundingBox inputBb; + inputBb.left = 0; + inputBb.top = 0; + inputBb.width = color.Width(); + inputBb.height = color.Height(); + + if (!loopyCachedImageAssign(_panorama, masked, panoramaBb, inputBb)) { return false; } return true; } - virtual bool initialize(image::TileCacheManager::shared_ptr & cacheManager) { + virtual bool initialize() { - if(!_panorama.createImage(cacheManager, _panoramaWidth, _panoramaHeight)) + if(!_panorama.createImage(_cacheManager, _panoramaWidth, _panoramaHeight)) { return false; } @@ -203,7 +220,7 @@ class Compositer } protected: - + image::TileCacheManager::shared_ptr _cacheManager; CachedImage _panorama; int _panoramaWidth; int _panoramaHeight; diff --git a/src/aliceVision/panorama/imageOps.cpp b/src/aliceVision/panorama/imageOps.cpp index a8753794f2..8b0bc75a75 100644 --- a/src/aliceVision/panorama/imageOps.cpp +++ b/src/aliceVision/panorama/imageOps.cpp @@ -9,13 +9,13 @@ void removeNegativeValues(CachedImage & img) [](const image::RGBfColor & c) -> image::RGBfColor { image::RGBfColor rpix; - image::RGBfColor ret; + image::RGBfColor ret = c; - rpix.r() = std::exp(c.r()); - rpix.g() = std::exp(c.g()); - rpix.b() = std::exp(c.b()); + rpix.r() = /*std::exp*/(c.r()); + rpix.g() = /*std::exp*/(c.g()); + rpix.b() = /*std::exp*/(c.b()); - if(rpix.r() < 0.0) + if (rpix.r() < 0.0) { ret.r() = 0.0; } diff --git a/src/aliceVision/panorama/laplacianCompositer.hpp b/src/aliceVision/panorama/laplacianCompositer.hpp index b067a9a971..345c692905 100644 --- a/src/aliceVision/panorama/laplacianCompositer.hpp +++ b/src/aliceVision/panorama/laplacianCompositer.hpp @@ -70,30 +70,25 @@ bool makeImagePyramidCompatible(image::Image& output, class LaplacianCompositer : public Compositer { public: - LaplacianCompositer(size_t outputWidth, size_t outputHeight, size_t bands) - : Compositer(outputWidth, outputHeight) - , _pyramid_panorama(outputWidth, outputHeight, bands) + LaplacianCompositer(image::TileCacheManager::shared_ptr & cacheManager, size_t outputWidth, size_t outputHeight, size_t bands) + : Compositer(cacheManager, outputWidth, outputHeight) + , _pyramidPanorama(outputWidth, outputHeight, bands) , _bands(bands) { } - virtual bool initialize(image::TileCacheManager::shared_ptr & cacheManager) + virtual bool initialize() { - if (!Compositer::initialize(cacheManager)) + if (!Compositer::initialize()) { return false; } - return _pyramid_panorama.initialize(cacheManager); + return _pyramidPanorama.initialize(_cacheManager); } - virtual bool append(const aliceVision::image::Image& color, - const aliceVision::image::Image& inputMask, - const aliceVision::image::Image& inputWeights, size_t offset_x, size_t offset_y) + size_t getOptimalScale(int width, int height) { - /*Get smallest size*/ - size_t minsize = std::min(color.Height(), color.Width()); - /* Look for the smallest scale such that the image is not smaller than the convolution window size. @@ -101,9 +96,22 @@ class LaplacianCompositer : public Compositer minsize / 5 = 2^x x = log2(minsize/5) */ + + size_t minsize = std::min(width, height); const float gaussian_filter_size = 5.0f; size_t optimal_scale = size_t(floor(std::log2(double(minsize) / gaussian_filter_size))); - if(optimal_scale < _bands) + + return optimal_scale; + } + + + virtual bool append(const aliceVision::image::Image& color, + const aliceVision::image::Image& inputMask, + const aliceVision::image::Image& inputWeights, + size_t offset_x, size_t offset_y, const BoundingBox & contentBox) + { + size_t optimalScale = getOptimalScale(contentBox.width, contentBox.height); + if(optimalScale < _bands) { ALICEVISION_LOG_ERROR("Decreasing scale !"); return false; @@ -111,10 +119,10 @@ class LaplacianCompositer : public Compositer //If the input scale is more important than previously processed, // The pyramid must be deepened accordingly - if(optimal_scale > _bands) + if(optimalScale > _bands) { - //_bands = optimal_scale; - //_pyramid_panorama.augment(_bands); + _bands = optimalScale; + _pyramidPanorama.augment(_cacheManager, _bands); } // Make sure input is compatible with pyramid processing @@ -144,7 +152,7 @@ class LaplacianCompositer : public Compositer } }*/ - _pyramid_panorama.apply(feathered, mask_pot, weights_pot, new_offset_x, new_offset_y); + _pyramidPanorama.apply(feathered, mask_pot, weights_pot, new_offset_x, new_offset_y); return true; } @@ -152,9 +160,9 @@ class LaplacianCompositer : public Compositer virtual bool terminate() { - _pyramid_panorama.rebuild(_panorama); + _pyramidPanorama.rebuild(_panorama); - _panorama.perPixelOperation( + /*_panorama.perPixelOperation( [](const image::RGBAfColor & a) -> image::RGBAfColor { image::RGBAfColor out; @@ -165,13 +173,13 @@ class LaplacianCompositer : public Compositer return out; } - ); + );*/ return true; } protected: - LaplacianPyramid _pyramid_panorama; + LaplacianPyramid _pyramidPanorama; size_t _bands; }; diff --git a/src/aliceVision/panorama/laplacianPyramid.cpp b/src/aliceVision/panorama/laplacianPyramid.cpp index 2e92b2cc09..b6fda7d965 100644 --- a/src/aliceVision/panorama/laplacianPyramid.cpp +++ b/src/aliceVision/panorama/laplacianPyramid.cpp @@ -14,6 +14,8 @@ _maxLevels(max_levels) { } + + bool LaplacianPyramid::initialize(image::TileCacheManager::shared_ptr & cacheManager) { size_t width = _baseWidth; @@ -38,22 +40,12 @@ bool LaplacianPyramid::initialize(image::TileCacheManager::shared_ptr & cacheMan return false; } - if(!color.perPixelOperation( - [](image::RGBfColor ) -> image::RGBfColor - { - return image::RGBfColor(0.0f, 0.0f, 0.0f); - }) - ) + if(!color.fill(image::RGBfColor(0.0f, 0.0f, 0.0f))) { return false; } - if(!weights.perPixelOperation( - [](float) -> float - { - return 0.0f; - }) - ) + if(!weights.fill(0.0f)) { return false; } @@ -68,98 +60,111 @@ bool LaplacianPyramid::initialize(image::TileCacheManager::shared_ptr & cacheMan return true; } -bool LaplacianPyramid::augment(size_t new_max_levels) +bool LaplacianPyramid::augment(image::TileCacheManager::shared_ptr & cacheManager, size_t newMaxLevels) { - - /*if(new_max_levels <= _levels.size()) + if(newMaxLevels <= _levels.size()) { return false; } + _maxLevels = newMaxLevels; - _maxLevels = new_max_levels; + //Get content of last level of pyramid + CachedImage lastColor = _levels[_levels.size() - 1]; + CachedImage largerWeight = _weights[_weights.size() - 1]; - size_t old_max_level = _levels.size(); + //Remove last level + /*_levels.pop_back(); + _weights.pop_back();*/ + + //Last level was multiplied by the weight. + //Remove this factor + lastColor.perPixelOperation(largerWeight, + [](const image::RGBfColor & c, const float & w) -> image::RGBfColor + { + if (w < 1e-6) + { + return image::RGBfColor(0.0f, 0.0f, 0.0f); + } - image::Image current_color = _levels[_levels.size() - 1]; - image::Image current_weights = _weights[_weights.size() - 1]; + image::RGBfColor r; - _levels[_levels.size() - 1].fill(image::RGBfColor(0.0f, 0.0f, 0.0f)); - _weights[_weights.size() - 1].fill(0.0f); + r.r() = c.r() / w; + r.g() = c.g() / w; + r.b() = c.b() / w; - image::Image current_mask(current_color.Width(), current_color.Height(), true, 0); - image::Image current_color_feathered(current_color.Width(), current_color.Height()); + return r; + } + ); - for(int i = 0; i < current_color.Height(); i++) + //Create a mask + CachedImage largerMask; + if(!largerMask.createImage(cacheManager, largerWeight.getWidth(), largerWeight.getHeight())) { - for(int j = 0; j < current_color.Width(); j++) + return false; + } + + //Build the mask + largerMask.perPixelOperation(largerWeight, + [](const unsigned char & c, const float & w) -> unsigned char { - if(current_weights(i, j) < 1e-6) + if (w < 1e-6) { - current_color(i, j) = image::RGBfColor(0.0); - continue; + return 0; } - current_color(i, j).r() = current_color(i, j).r() / current_weights(i, j); - current_color(i, j).g() = current_color(i, j).g() / current_weights(i, j); - current_color(i, j).b() = current_color(i, j).b() / current_weights(i, j); - current_mask(i, j) = 255; + return 255; } - } + ); - feathering(current_color_feathered, current_color, current_mask); - current_color = current_color_feathered; + largerMask.writeImage("/home/mmoc/mask.exr"); - for(int l = old_max_level; l < new_max_levels; l++) - { - - _levels.emplace_back(_levels[l - 1].Width() / 2, _levels[l - 1].Height() / 2, true, - image::RGBfColor(0.0f, 0.0f, 0.0f)); - _weights.emplace_back(_weights[l - 1].Width() / 2, _weights[l - 1].Height() / 2, true, 0.0f); - } - int width = current_color.Width(); - int height = current_color.Height(); - image::Image next_color; - image::Image next_weights; + /*int largerLevel = _levels.size() - 1; + int currentLevel = _levels.size(); - for(int l = old_max_level - 1; l < new_max_levels - 1; l++) - { - aliceVision::image::Image buf(width, height); - aliceVision::image::Image buf2(width, height); - aliceVision::image::Image bufw(width, height); + CachedImage largerColor = _levels[largerLevel]; + CachedImage largerWeight = _weights[largerLevel]; - next_color = aliceVision::image::Image(width / 2, height / 2); - next_weights = aliceVision::image::Image(width / 2, height / 2); + int width = largerColor.getWidth(); + int height = largerColor.getHeight(); - convolveGaussian5x5(buf, current_color); - downscale(next_color, buf); + CachedImage color; + if(!color.createImage(cacheManager, width, height)) + { + return false; + } - convolveGaussian5x5(bufw, current_weights); - downscale(next_weights, bufw); + CachedImage weights; + if(!weights.createImage(cacheManager, width, height)) + { + return false; + } - upscale(buf, next_color); - convolveGaussian5x5(buf2, buf); + aliceVision::image::Image extractedColor(width, height); + aliceVision::image::Image extractedWeight(width, height); - for(int i = 0; i < buf2.Height(); i++) - { - for(int j = 0; j < buf2.Width(); j++) - { - buf2(i, j) *= 4.0f; - } - } + BoundingBox extractBb; + extractBb.left = 0; + extractBb.top = 0; + extractBb.width = width; + extractBb.height = height; - substract(current_color, current_color, buf2); - merge(current_color, current_weights, l, 0, 0); + if (!loopyCachedImageExtract(extractedColor, largerColor, extractBb)) + { + return false; + } - current_color = next_color; - current_weights = next_weights; - width /= 2; - height /= 2; + if (!loopyCachedImageExtract(extractedWeight, largerWeight, extractBb)) + { + return false; } - merge(current_color, current_weights, _levels.size() - 1, 0, 0);*/ + //image was multiplied with a weight, we need to get back the original weight + _levels.push_back(color); + _weights.push_back(weights);*/ + return true; } @@ -167,9 +172,6 @@ bool LaplacianPyramid::apply(const aliceVision::image::Image& const aliceVision::image::Image& mask, const aliceVision::image::Image& weights, size_t offset_x, size_t offset_y) { - - - int width = source.Width(); int height = source.Height(); @@ -197,6 +199,7 @@ bool LaplacianPyramid::apply(const aliceVision::image::Image& image::Image current_mask = mask_float; image::Image next_mask; + for(int l = 0; l < _levels.size() - 1; l++) { aliceVision::image::Image buf_masked(width, height); @@ -333,11 +336,18 @@ bool LaplacianPyramid::merge(const aliceVision::image::Image& } } - if (!loopyCachedImageAssign(img, extractedColor, extractBb)) { + BoundingBox inputBb; + inputBb.left = 0; + inputBb.top = 0; + inputBb.width = extractedColor.Width(); + inputBb.height = extractedColor.Height(); + + if (!loopyCachedImageAssign(img, extractedColor, extractBb, inputBb)) { return false; } - if (!loopyCachedImageAssign(weight, extractedWeight, extractBb)) { + + if (!loopyCachedImageAssign(weight, extractedWeight, extractBb, inputBb)) { return false; } @@ -347,6 +357,7 @@ bool LaplacianPyramid::merge(const aliceVision::image::Image& bool LaplacianPyramid::rebuild(CachedImage& output) { + // We first want to compute the final pixels mean for(int l = 0; l < _levels.size(); l++) { _levels[l].perPixelOperation(_weights[l], @@ -373,46 +384,130 @@ bool LaplacianPyramid::rebuild(CachedImage& output) for(int l = _levels.size() - 2; l >= 0; l--) { + const size_t processingSize = 512; + const size_t borderSize = 5; - /*aliceVision::image::Image buf(_levels[l].Width(), _levels[l].Height()); - aliceVision::image::Image buf2(_levels[l].Width(), _levels[l].Height()); + int halfLevel = l + 1; + int currentLevel = l; - upscale(buf, _levels[l + 1]); - convolveGaussian5x5(buf2, buf, true); - - for(int i = 0; i < buf2.Height(); i++) + int x = 0; + int y = 0; + + + for (int y = 0; y < _levels[halfLevel].getHeight(); y += processingSize) { - for(int j = 0; j < buf2.Width(); j++) + for (int x = 0; x < _levels[halfLevel].getWidth(); x += processingSize) { - buf2(i, j) *= 4.0f; + BoundingBox extractedBb; + extractedBb.left = x; + extractedBb.top = y; + extractedBb.width = processingSize; + extractedBb.height = processingSize; + extractedBb.clampLeft(); + extractedBb.clampTop(); + extractedBb.clampRight(_levels[halfLevel].getWidth() - 1); + extractedBb.clampBottom(_levels[halfLevel].getHeight() - 1); + + BoundingBox dilatedBb = extractedBb.dilate(borderSize); + dilatedBb.clampLeft(); + dilatedBb.clampTop(); + dilatedBb.clampBottom(_levels[halfLevel].getHeight() - 1); + + BoundingBox doubleDilatedBb = dilatedBb.doubleSize(); + BoundingBox doubleBb = extractedBb.doubleSize(); + + aliceVision::image::Image extracted(dilatedBb.width, dilatedBb.height); + if (!loopyCachedImageExtract(extracted, _levels[halfLevel], dilatedBb)) + { + return false; + } + + aliceVision::image::Image extractedNext(doubleDilatedBb.width, doubleDilatedBb.height); + if (!loopyCachedImageExtract(extractedNext, _levels[currentLevel], doubleDilatedBb)) + { + return false; + } + + aliceVision::image::Image buf(doubleDilatedBb.width, doubleDilatedBb.height); + aliceVision::image::Image buf2(doubleDilatedBb.width, doubleDilatedBb.height); + + upscale(buf, extracted); + convolveGaussian5x5(buf2, buf, false); + + for(int i = 0; i < buf2.Height(); i++) + { + for(int j = 0; j < buf2.Width(); j++) + { + buf2(i, j) *= 4.0f; + } + } + + addition(extractedNext, extractedNext, buf2); + + BoundingBox inputBb; + inputBb.left = doubleBb.left - doubleDilatedBb.left; + inputBb.top = doubleBb.top - doubleDilatedBb.top; + inputBb.width = doubleBb.width; + inputBb.height = doubleBb.height; + + if (!loopyCachedImageAssign(_levels[currentLevel], extractedNext, doubleBb, inputBb)) + { + return false; + } } } - addition(_levels[l], _levels[l], buf2);*/ - removeNegativeValues(_levels[l]); + removeNegativeValues(_levels[currentLevel]); } - // Write output to RGBA - /*for(int i = 0; i < output.Height(); i++) + for(int i = 0; i < output.getTiles().size(); i++) { - for(int j = 0; j < output.Width(); j++) + + std::vector & rowOutput = output.getTiles()[i]; + std::vector & rowInput = _levels[0].getTiles()[i]; + std::vector & rowWeight = _weights[0].getTiles()[i]; + + for(int j = 0; j < rowOutput.size(); j++) { - output(i, j).r() = _levels[0](i, j).r(); - output(i, j).g() = _levels[0](i, j).g(); - output(i, j).b() = _levels[0](i, j).b(); - if(_weights[0](i, j) < 1e-6) + if(!rowOutput[j]->acquire()) { - output(i, j).a() = 0.0f; + return false; } - else + + if(!rowInput[j]->acquire()) + { + return false; + } + + if(!rowWeight[j]->acquire()) { - output(i, j).a() = 1.0f; + return false; + } + + image::RGBAfColor* ptrOutput = (image::RGBAfColor *)rowOutput[j]->getDataPointer(); + image::RGBfColor* ptrInput = (image::RGBfColor *)rowInput[j]->getDataPointer(); + float* ptrWeight = (float *)rowWeight[j]->getDataPointer(); + + for (int k = 0; k < output.getTileSize() * output.getTileSize(); k++) + { + ptrOutput[k].r() = ptrInput[k].r(); + ptrOutput[k].g() = ptrInput[k].g(); + ptrOutput[k].b() = ptrInput[k].b(); + + if(ptrWeight[k] < 1e-6) + { + ptrOutput[k].a() = 1.0f; + } + else + { + ptrOutput[k].a() = 1.0f; + } } } - }*/ + } return true; } -} // namespace aliceVision \ No newline at end of file +} // namespace aliceVision diff --git a/src/aliceVision/panorama/laplacianPyramid.hpp b/src/aliceVision/panorama/laplacianPyramid.hpp index 804ef0d53e..0faa5ae399 100644 --- a/src/aliceVision/panorama/laplacianPyramid.hpp +++ b/src/aliceVision/panorama/laplacianPyramid.hpp @@ -14,7 +14,7 @@ class LaplacianPyramid bool initialize(image::TileCacheManager::shared_ptr & cacheManager); - bool augment(size_t new_max_levels); + bool augment(image::TileCacheManager::shared_ptr & cacheManager, size_t new_max_levels); bool apply(const aliceVision::image::Image& source, const aliceVision::image::Image& mask, const aliceVision::image::Image& weights, diff --git a/src/aliceVision/panorama/remapBbox.cpp b/src/aliceVision/panorama/remapBbox.cpp index fe2d967c64..254152189a 100644 --- a/src/aliceVision/panorama/remapBbox.cpp +++ b/src/aliceVision/panorama/remapBbox.cpp @@ -324,7 +324,6 @@ bool computeCoarseBB_Pinhole(BoundingBox& coarse_bbox, const std::pair } else if(crossH) { - int first_cross = 0; for(int i = 0; i < 8; i++) { @@ -380,7 +379,7 @@ bool computeCoarseBB_Pinhole(BoundingBox& coarse_bbox, const std::pair } } } - + bbox_width = bbox_right + (panoramaSize.first - bbox_left); } else diff --git a/src/aliceVision/panorama/seams.cpp b/src/aliceVision/panorama/seams.cpp index ae70ab6883..ddd52b2acf 100644 --- a/src/aliceVision/panorama/seams.cpp +++ b/src/aliceVision/panorama/seams.cpp @@ -215,11 +215,17 @@ bool WTASeams::append(const aliceVision::image::Image& inputMask, } } - if (!loopyCachedImageAssign(_weights, weights, globalBb)) { + BoundingBox inputBb; + inputBb.left = 0; + inputBb.top = 0; + inputBb.width = labels.Width(); + inputBb.height = labels.Height(); + + if (!loopyCachedImageAssign(_weights, weights, globalBb, inputBb)) { return false; } - if (!loopyCachedImageAssign(_labels, labels, globalBb)) { + if (!loopyCachedImageAssign(_labels, labels, globalBb, inputBb)) { return false; } diff --git a/src/software/pipeline/main_panoramaCompositing.cpp b/src/software/pipeline/main_panoramaCompositing.cpp index 31624ce597..dd5168a1fd 100644 --- a/src/software/pipeline/main_panoramaCompositing.cpp +++ b/src/software/pipeline/main_panoramaCompositing.cpp @@ -278,8 +278,13 @@ int aliceVision_main(int argc, char** argv) // Configure the cache manager memory cacheManager->setInCoreMaxObjectCount(1000); - LaplacianCompositer compositer(panoramaSize.first, panoramaSize.second, 3); - compositer.initialize(cacheManager); + LaplacianCompositer compositer(cacheManager, panoramaSize.first, panoramaSize.second, 1); + + if (!compositer.initialize()) + { + ALICEVISION_LOG_ERROR("Failed to initialize compositer"); + return EXIT_FAILURE; + } CachedImage labels; if (!computeWTALabels(labels, cacheManager, sfmData, warpingFolder, panoramaSize)) @@ -295,11 +300,34 @@ int aliceVision_main(int argc, char** argv) }*/ - for(const auto& it : sfmData.getViews()) + //Get a list of views ordered by their image scale + std::vector> viewOrderedByScale; { - IndexT viewId = it.first; - auto view = it.second; + std::map>> mapViewsScale; + for(const auto& it : sfmData.getViews()) + { + const std::string maskPath = (fs::path(warpingFolder) / (std::to_string(it.first) + "_mask.exr")).string(); + oiio::ParamValueList metadata = image::readImageMetadata(maskPath); + + const std::size_t contentW = metadata.find("AliceVision:contentW")->get_int(); + const std::size_t contentH = metadata.find("AliceVision:contentH")->get_int(); + + size_t scale = compositer.getOptimalScale(contentW, contentH); + mapViewsScale[scale].push_back(it.second); + } + for (auto scaledList : mapViewsScale) + { + for (auto item : scaledList.second) + { + viewOrderedByScale.push_back(item); + } + } + } + + for(const auto & view : viewOrderedByScale) + { + IndexT viewId = view->getViewId(); if(!sfmData.isPoseAndIntrinsicDefined(view.get())) { // skip unreconstructed views @@ -315,6 +343,10 @@ int aliceVision_main(int argc, char** argv) oiio::ParamValueList metadata = image::readImageMetadata(imagePath); const std::size_t offsetX = metadata.find("AliceVision:offsetX")->get_int(); const std::size_t offsetY = metadata.find("AliceVision:offsetY")->get_int(); + const std::size_t contentX = metadata.find("AliceVision:contentX")->get_int(); + const std::size_t contentY = metadata.find("AliceVision:contentY")->get_int(); + const std::size_t contentW = metadata.find("AliceVision:contentW")->get_int(); + const std::size_t contentH = metadata.find("AliceVision:contentH")->get_int(); // Load mask const std::string maskPath = (fs::path(warpingFolder) / (std::to_string(viewId) + "_mask.exr")).string(); @@ -328,7 +360,12 @@ int aliceVision_main(int argc, char** argv) image::Image weights; image::readImage(weightsPath, weights, image::EImageColorSpace::NO_CONVERSION);*/ - + BoundingBox imageContent; + imageContent.left = contentX; + imageContent.top = contentY; + imageContent.width = contentW; + imageContent.height = contentH; + image::Image seams(mask.Width(), mask.Height()); if (!getMaskFromLabels(seams, labels, viewId, offsetX, offsetY)) { @@ -336,7 +373,7 @@ int aliceVision_main(int argc, char** argv) return EXIT_FAILURE; } - compositer.append(source, mask, seams, offsetX, offsetY); + compositer.append(source, mask, seams, offsetX, offsetY, imageContent); } compositer.terminate(); From 3f6cba48ab89e0e3c714ca886ec90c0f262fa3d8 Mon Sep 17 00:00:00 2001 From: Fabien Servant Date: Tue, 6 Oct 2020 17:18:05 +0200 Subject: [PATCH 08/79] [panorama] wip compositing --- src/aliceVision/panorama/cachedImage.cpp | 43 +++ src/aliceVision/panorama/cachedImage.hpp | 63 ++++ src/aliceVision/panorama/feathering.cpp | 276 ++++++++++++++++++ src/aliceVision/panorama/feathering.hpp | 5 + .../panorama/laplacianCompositer.hpp | 22 +- src/aliceVision/panorama/laplacianPyramid.cpp | 233 ++++++++++++--- src/aliceVision/panorama/warper.cpp | 20 +- .../pipeline/main_panoramaCompositing.cpp | 2 +- .../pipeline/main_panoramaWarping.cpp | 4 +- 9 files changed, 617 insertions(+), 51 deletions(-) diff --git a/src/aliceVision/panorama/cachedImage.cpp b/src/aliceVision/panorama/cachedImage.cpp index 83b7803214..b146d9512a 100644 --- a/src/aliceVision/panorama/cachedImage.cpp +++ b/src/aliceVision/panorama/cachedImage.cpp @@ -132,6 +132,49 @@ bool CachedImage::writeImage(const std::string& path) return true; } +template <> +bool CachedImage::writeImage(const std::string& path) +{ + + std::unique_ptr out = oiio::ImageOutput::create(path); + if(!out) + { + return false; + } + + oiio::ImageSpec spec(_memoryWidth, _memoryHeight, 1, oiio::TypeDesc::FLOAT); + spec.tile_width = _tileSize; + spec.tile_height = _tileSize; + + if(!out->open(path, spec)) + { + return false; + } + + for(int i = 0; i < _tilesArray.size(); i++) + { + + std::vector& row = _tilesArray[i]; + + for(int j = 0; j < row.size(); j++) + { + + if(!row[j]->acquire()) + { + return false; + } + + unsigned char* ptr = row[j]->getDataPointer(); + + out->write_tile(j * _tileSize, i * _tileSize, 0, oiio::TypeDesc::FLOAT, ptr); + } + } + + out->close(); + + return true; +} + template <> bool CachedImage::writeImage(const std::string& path) { diff --git a/src/aliceVision/panorama/cachedImage.hpp b/src/aliceVision/panorama/cachedImage.hpp index 47e1e162a6..3bf6034f33 100644 --- a/src/aliceVision/panorama/cachedImage.hpp +++ b/src/aliceVision/panorama/cachedImage.hpp @@ -376,6 +376,67 @@ class CachedImage return true; } + static bool getTileAsImage(image::Image & ret, image::CachedTile::smart_pointer tile) + { + if(!tile) + { + return false; + } + + if(!tile->acquire()) + { + return false; + } + + ret.resize(tile->getTileWidth(), tile->getTileHeight()); + T * data = (T*)tile->getDataPointer(); + for (int i = 0; i < tile->getTileHeight(); i++) + { + for (int j = 0; j < tile->getTileWidth(); j++) + { + ret(i, j) = *data; + data++; + } + } + + return true; + } + + static bool setTileWithImage(image::CachedTile::smart_pointer tile, const image::Image & ret) + { + if (ret.Width() != tile->getTileWidth()) + { + return false; + } + + if (ret.Height() != tile->getTileHeight()) + { + return false; + } + + if(!tile) + { + return false; + } + + if(!tile->acquire()) + { + return false; + } + + T * data = (T*)tile->getDataPointer(); + for (int i = 0; i < tile->getTileHeight(); i++) + { + for (int j = 0; j < tile->getTileWidth(); j++) + { + *data = ret(i, j); + data++; + } + } + + return true; + } + bool fill(const T & val) { if (!perPixelOperation( @@ -418,6 +479,8 @@ bool CachedImage::writeImage(const std::string& path); template <> bool CachedImage::writeImage(const std::string& path); +template <> +bool CachedImage::writeImage(const std::string& path); template <> bool CachedImage::writeImage(const std::string& path); diff --git a/src/aliceVision/panorama/feathering.cpp b/src/aliceVision/panorama/feathering.cpp index 867a7c0e5a..ab49bd8a3c 100644 --- a/src/aliceVision/panorama/feathering.cpp +++ b/src/aliceVision/panorama/feathering.cpp @@ -123,4 +123,280 @@ bool feathering(aliceVision::image::Image& output, return true; } + +bool feathering(CachedImage & input_output, CachedImage & inputMask) +{ + if (input_output.getTileSize() < 2) + { + return false; + } + + if (input_output.getTileSize() != inputMask.getTileSize()) + { + return false; + } + + if (input_output.getWidth() != inputMask.getWidth()) + { + return false; + } + + if (input_output.getHeight() != inputMask.getHeight()) + { + return false; + } + + std::vector> & tilesColor = input_output.getTiles(); + std::vector> & tilesMask = inputMask.getTiles(); + + if (tilesColor.size() == 0) + { + return false; + } + + int gridHeight = tilesColor.size(); + int gridWidth = tilesColor[0].size(); + int currentSize = input_output.getTileSize(); + + //Make sure the grid has a pow2 size, and is square + gridWidth = pow(2.0, std::ceil(std::log2(float(gridWidth)))); + gridHeight = pow(2.0, std::ceil(std::log2(float(gridHeight)))); + int gridSize = std::max(gridWidth, gridHeight); + + image::Image colorTile; + image::Image maskTile; + + image::Image featheredGrid(gridSize, gridSize); + image::Image colorGrid(gridSize, gridSize); + image::Image maskGrid(gridSize, gridSize, true, 0); + + /*Build the grid color image */ + for (int i = 0; i < tilesColor.size(); i++) + { + std::vector rowColor = tilesColor[i]; + std::vector rowMask = tilesMask[i]; + + for (int j = 0; j < rowColor.size(); j++) + { + if (!CachedImage::getTileAsImage(colorTile, rowColor[j])) + { + return false; + } + + if (!CachedImage::getTileAsImage(maskTile, rowMask[j])) + { + return false; + } + + while (1) + { + image::Image smallerTile(colorTile.Width() / 2, colorTile.Height() / 2); + image::Image smallerMask(maskTile.Width() / 2, maskTile.Height() / 2); + + for(int y = 0; y < smallerTile.Height(); y++) + { + int dy = y * 2; + for(int x = 0; x < smallerTile.Width(); x++) + { + int dx = x * 2; + + int count = 0; + + smallerTile(y, x) = image::RGBfColor(0.0, 0.0, 0.0); + + if(maskTile(dy, dx)) + { + smallerTile(y, x) += colorTile(dy, dx); + count++; + } + + if(maskTile(dy, dx + 1)) + { + smallerTile(y, x) += colorTile(dy, dx + 1); + count++; + } + + if(maskTile(dy + 1, dx)) + { + smallerTile(y, x) += colorTile(dy + 1, dx); + count++; + } + + if(maskTile(dy + 1, dx + 1)) + { + smallerTile(y, x) += colorTile(dy + 1, dx + 1); + count++; + } + + if(count > 0) + { + smallerTile(y, x) /= float(count); + smallerMask(y, x) = 1; + } + else + { + smallerMask(y, x) = 0; + } + } + } + + colorTile = smallerTile; + maskTile = smallerMask; + if (colorTile.Width() < 2 || colorTile.Height() < 2) + { + break; + } + } + + maskGrid(i, j) = maskTile(0, 0); + colorGrid(i, j) = colorTile(0, 0); + } + } + + if (!feathering(featheredGrid, colorGrid, maskGrid)) + { + return false; + } + + for (int i = 0; i < tilesColor.size(); i++) + { + std::vector rowColor = tilesColor[i]; + std::vector rowMask = tilesMask[i]; + + for (int j = 0; j < rowColor.size(); j++) + { + if (!CachedImage::getTileAsImage(colorTile, rowColor[j])) + { + return false; + } + + if (!CachedImage::getTileAsImage(maskTile, rowMask[j])) + { + return false; + } + + std::vector> pyramid_colors; + std::vector> pyramid_masks; + + pyramid_colors.push_back(colorTile); + pyramid_masks.push_back(maskTile); + + while (1) + { + image::Image & largerTile = pyramid_colors[pyramid_colors.size() - 1]; + image::Image & largerMask = pyramid_masks[pyramid_masks.size() - 1]; + + image::Image smallerTile(largerTile.Width() / 2, largerTile.Height() / 2); + image::Image smallerMask(largerMask.Width() / 2, largerMask.Height() / 2); + + for(int y = 0; y < smallerTile.Height(); y++) + { + int dy = y * 2; + for(int x = 0; x < smallerTile.Width(); x++) + { + int dx = x * 2; + + int count = 0; + + smallerTile(y, x) = image::RGBfColor(0.0, 0.0, 0.0); + + if(largerMask(dy, dx)) + { + smallerTile(y, x) += largerTile(dy, dx); + count++; + } + + if(largerMask(dy, dx + 1)) + { + smallerTile(y, x) += largerTile(dy, dx + 1); + count++; + } + + if(largerMask(dy + 1, dx)) + { + smallerTile(y, x) += largerTile(dy + 1, dx); + count++; + } + + if(largerMask(dy + 1, dx + 1)) + { + smallerTile(y, x) += largerTile(dy + 1, dx + 1); + count++; + } + + if(count > 0) + { + smallerTile(y, x) /= float(count); + smallerMask(y, x) = 1; + } + else + { + smallerMask(y, x) = 0; + } + } + } + + + pyramid_colors.push_back(smallerTile); + pyramid_masks.push_back(smallerMask); + + if (smallerTile.Width() < 2 || smallerTile.Height() < 2) + { + break; + } + } + + image::Image & img = pyramid_colors[pyramid_colors.size() - 1]; + image::Image & mask = pyramid_masks[pyramid_masks.size() - 1]; + + if (!mask(0, 0)) + { + mask(0, 0) = 255; + img(0, 0) = featheredGrid(i, j); + } + + for(int lvl = pyramid_colors.size() - 2; lvl >= 0; lvl--) + { + + image::Image & src = pyramid_colors[lvl]; + image::Image & src_mask = pyramid_masks[lvl]; + image::Image & ref = pyramid_colors[lvl + 1]; + image::Image & ref_mask = pyramid_masks[lvl + 1]; + + for(int i = 0; i < src_mask.Height(); i++) + { + for(int j = 0; j < src_mask.Width(); j++) + { + if(!src_mask(i, j)) + { + int mi = i / 2; + int mj = j / 2; + + if(mi >= ref_mask.Height()) + { + mi = ref_mask.Height() - 1; + } + + if(mj >= ref_mask.Width()) + { + mj = ref_mask.Width() - 1; + } + + src_mask(i, j) = ref_mask(mi, mj); + src(i, j) = ref(mi, mj); + } + } + } + } + + if (!CachedImage::setTileWithImage(rowColor[j], pyramid_colors[0])) + { + return false; + } + } + } + + return true; +} + } // namespace aliceVision \ No newline at end of file diff --git a/src/aliceVision/panorama/feathering.hpp b/src/aliceVision/panorama/feathering.hpp index 8146b91177..028b98d085 100644 --- a/src/aliceVision/panorama/feathering.hpp +++ b/src/aliceVision/panorama/feathering.hpp @@ -2,6 +2,8 @@ #include +#include + namespace aliceVision { @@ -9,4 +11,7 @@ bool feathering(aliceVision::image::Image& output, const aliceVision::image::Image& color, const aliceVision::image::Image& inputMask); +bool feathering(CachedImage& input_output, + CachedImage& inputMask); + } \ No newline at end of file diff --git a/src/aliceVision/panorama/laplacianCompositer.hpp b/src/aliceVision/panorama/laplacianCompositer.hpp index 345c692905..5102e125b7 100644 --- a/src/aliceVision/panorama/laplacianCompositer.hpp +++ b/src/aliceVision/panorama/laplacianCompositer.hpp @@ -101,10 +101,12 @@ class LaplacianCompositer : public Compositer const float gaussian_filter_size = 5.0f; size_t optimal_scale = size_t(floor(std::log2(double(minsize) / gaussian_filter_size))); + optimal_scale = 5; + return optimal_scale; } - + virtual bool append(const aliceVision::image::Image& color, const aliceVision::image::Image& inputMask, const aliceVision::image::Image& inputWeights, @@ -135,11 +137,13 @@ class LaplacianCompositer : public Compositer makeImagePyramidCompatible(mask_pot, new_offset_x, new_offset_y, inputMask, offset_x, offset_y, _bands); makeImagePyramidCompatible(weights_pot, new_offset_x, new_offset_y, inputWeights, offset_x, offset_y, _bands); - //std::cout << new_offset_x << " " << new_offset_y << std::endl; - + // Fill Color images masked parts with fake but coherent info aliceVision::image::Image feathered; - feathering(feathered, color_pot, mask_pot); + if (!feathering(feathered, color_pot, mask_pot)) + { + return false; + } /*To log space for hdr*/ /*for(int i = 0; i < feathered.Height(); i++) @@ -152,7 +156,10 @@ class LaplacianCompositer : public Compositer } }*/ - _pyramidPanorama.apply(feathered, mask_pot, weights_pot, new_offset_x, new_offset_y); + if (!_pyramidPanorama.apply(feathered, mask_pot, weights_pot, new_offset_x, new_offset_y)) + { + return false; + } return true; } @@ -160,7 +167,10 @@ class LaplacianCompositer : public Compositer virtual bool terminate() { - _pyramidPanorama.rebuild(_panorama); + if (!_pyramidPanorama.rebuild(_panorama)) + { + return false; + } /*_panorama.perPixelOperation( [](const image::RGBAfColor & a) -> image::RGBAfColor { diff --git a/src/aliceVision/panorama/laplacianPyramid.cpp b/src/aliceVision/panorama/laplacianPyramid.cpp index b6fda7d965..8713180ff0 100644 --- a/src/aliceVision/panorama/laplacianPyramid.cpp +++ b/src/aliceVision/panorama/laplacianPyramid.cpp @@ -62,23 +62,43 @@ bool LaplacianPyramid::initialize(image::TileCacheManager::shared_ptr & cacheMan bool LaplacianPyramid::augment(image::TileCacheManager::shared_ptr & cacheManager, size_t newMaxLevels) { + return true; + if(newMaxLevels <= _levels.size()) { return false; } + + int oldMaxLevels = _levels.size(); _maxLevels = newMaxLevels; + //Get content of last level of pyramid - CachedImage lastColor = _levels[_levels.size() - 1]; - CachedImage largerWeight = _weights[_weights.size() - 1]; + CachedImage largerColor; + if(!largerColor.createImage(cacheManager, _levels[oldMaxLevels - 1].getWidth(), _levels[oldMaxLevels - 1].getHeight())) + { + return false; + } - //Remove last level - /*_levels.pop_back(); - _weights.pop_back();*/ + if (!largerColor.deepCopy(_levels[oldMaxLevels - 1])) + { + return false; + } + + CachedImage largerWeight; + if(!largerWeight.createImage(cacheManager, _weights[oldMaxLevels - 1].getWidth(), _weights[oldMaxLevels - 1].getHeight())) + { + return false; + } + + if (!largerWeight.deepCopy(_weights[oldMaxLevels - 1])) + { + return false; + } //Last level was multiplied by the weight. //Remove this factor - lastColor.perPixelOperation(largerWeight, + largerColor.perPixelOperation(largerWeight, [](const image::RGBfColor & c, const float & w) -> image::RGBfColor { if (w < 1e-6) @@ -116,55 +136,185 @@ bool LaplacianPyramid::augment(image::TileCacheManager::shared_ptr & cacheManage } ); - largerMask.writeImage("/home/mmoc/mask.exr"); + if (!feathering(largerColor, largerMask)) + { + return false; + } - /*int largerLevel = _levels.size() - 1; - int currentLevel = _levels.size(); + //Augment the number of levels + for (int level = oldMaxLevels; level < newMaxLevels; level++) + { + CachedImage pyramidImage; + CachedImage pyramidWeights; - CachedImage largerColor = _levels[largerLevel]; - CachedImage largerWeight = _weights[largerLevel]; + if(!pyramidImage.createImage(cacheManager, _levels[_levels.size() - 1].getWidth() / 2, _levels[_levels.size() - 1].getHeight() / 2)) + { + return false; + } - int width = largerColor.getWidth(); - int height = largerColor.getHeight(); + if(!pyramidWeights.createImage(cacheManager, _weights[_weights.size() - 1].getWidth() / 2, _weights[_weights.size() - 1].getHeight() / 2)) + { + return false; + } + + pyramidImage.fill(image::RGBfColor(0.0f)); + pyramidWeights.fill(0.0f); - CachedImage color; - if(!color.createImage(cacheManager, width, height)) - { - return false; + _levels.push_back(pyramidImage); + _weights.push_back(pyramidWeights); } - CachedImage weights; - if(!weights.createImage(cacheManager, width, height)) + + const int processingSize = 512; + const size_t borderSize = 5; + + CachedImage currentImage = largerColor; + CachedImage currentWeights = largerWeight; + + for (int level = oldMaxLevels - 1; level < _maxLevels - 1; level++) { - return false; - } + CachedImage nextImage; + CachedImage nextWeights; - aliceVision::image::Image extractedColor(width, height); - aliceVision::image::Image extractedWeight(width, height); + if(!nextImage.createImage(cacheManager, currentImage.getWidth() / 2, currentImage.getHeight() / 2)) + { + return false; + } - BoundingBox extractBb; - extractBb.left = 0; - extractBb.top = 0; - extractBb.width = width; - extractBb.height = height; + if(!nextWeights.createImage(cacheManager, currentImage.getWidth() / 2, currentImage.getHeight() / 2)) + { + return false; + } + nextImage.fill(image::RGBfColor(0.0f)); + nextWeights.fill(0.0f); - if (!loopyCachedImageExtract(extractedColor, largerColor, extractBb)) - { - return false; - } + for (int y = 0; y < nextImage.getHeight(); y += processingSize) + { + for (int x = 0; x < nextImage.getWidth(); x += processingSize) + { + BoundingBox nextBbox; + nextBbox.left = x; + nextBbox.top = y; + nextBbox.width = processingSize; + nextBbox.height = processingSize; + nextBbox.clampRight(nextImage.getWidth() - 1); + nextBbox.clampBottom(nextImage.getHeight() - 1); + + BoundingBox dilatedNextBbox = nextBbox.dilate(borderSize); + dilatedNextBbox.clampLeft(); + dilatedNextBbox.clampTop(); + dilatedNextBbox.clampBottom(nextImage.getHeight() - 1); + + BoundingBox currentBbox = nextBbox.doubleSize(); + BoundingBox dilatedCurrentBbox = dilatedNextBbox.doubleSize(); + + aliceVision::image::Image extractedColor(dilatedCurrentBbox.width, dilatedCurrentBbox.height); + if (!loopyCachedImageExtract(extractedColor, currentImage, dilatedCurrentBbox)) + { + return false; + } - if (!loopyCachedImageExtract(extractedWeight, largerWeight, extractBb)) - { - return false; + aliceVision::image::Image extractedWeight(dilatedCurrentBbox.width, dilatedCurrentBbox.height); + if (!loopyCachedImageExtract(extractedWeight, currentWeights, dilatedCurrentBbox)) + { + return false; + } + + + /*Compute raw next scale from current scale */ + aliceVision::image::Image buf(dilatedCurrentBbox.width, dilatedCurrentBbox.height); + aliceVision::image::Image bufw(dilatedCurrentBbox.width, dilatedCurrentBbox.height); + aliceVision::image::Image colorDownscaled(dilatedNextBbox.width, dilatedNextBbox.height); + aliceVision::image::Image weightDownscaled(dilatedNextBbox.width, dilatedNextBbox.height); + + convolveGaussian5x5(buf, extractedColor); + convolveGaussian5x5(bufw, extractedWeight); + + downscale(colorDownscaled, buf); + downscale(weightDownscaled, bufw); + + BoundingBox saveBoundingBox; + saveBoundingBox.left = nextBbox.left - dilatedNextBbox.left; + saveBoundingBox.top = nextBbox.top - dilatedNextBbox.top; + saveBoundingBox.width = nextBbox.width; + saveBoundingBox.height = nextBbox.height; + + + + if (!loopyCachedImageAssign(nextImage, colorDownscaled, nextBbox, saveBoundingBox)) { + return false; + } + + if (!loopyCachedImageAssign(nextWeights, weightDownscaled, nextBbox, saveBoundingBox)) { + return false; + } + + + + /* Compute difference */ + aliceVision::image::Image buf2(dilatedCurrentBbox.width, dilatedCurrentBbox.height); + upscale(buf, colorDownscaled); + convolveGaussian5x5(buf2, buf); + + for (int i = 0; i < buf2.Height(); i++) { + for (int j = 0; j < buf2.Width(); j++) { + buf2(i,j) *= 4.0f; + } + } + + substract(extractedColor, extractedColor, buf2); + + for (int i = 0; i < extractedColor.Height(); i++) + { + for (int j = 0; j < extractedColor.Width(); j++) + { + extractedColor(i, j).r() = extractedColor(i, j).r() * extractedWeight(i, j); + extractedColor(i, j).g() = extractedColor(i, j).r() * extractedWeight(i, j); + extractedColor(i, j).b() = extractedColor(i, j).b() * extractedWeight(i, j); + } + } + + + saveBoundingBox.left = currentBbox.left - dilatedCurrentBbox.left; + saveBoundingBox.top = currentBbox.top - dilatedCurrentBbox.top; + saveBoundingBox.width = currentBbox.width; + saveBoundingBox.height = currentBbox.height; + + if (!loopyCachedImageAssign(_levels[level], extractedColor, currentBbox, saveBoundingBox)) { + return false; + } + + if (!loopyCachedImageAssign(_weights[level], extractedWeight, currentBbox, saveBoundingBox)) { + return false; + } + } + } + + currentImage = nextImage; + currentWeights = nextWeights; } - //image was multiplied with a weight, we need to get back the original weight - _levels.push_back(color); - _weights.push_back(weights);*/ - + currentImage.perPixelOperation(currentWeights, + [](const image::RGBfColor & c, const float & w) -> image::RGBfColor + { + image::RGBfColor r; + + r.r() = c.r() * w; + r.g() = c.g() * w; + r.b() = c.b() * w; + + return r; + } + ); + + + _levels[_levels.size() - 1].deepCopy(currentImage); + _weights[_weights.size() - 1].deepCopy(currentWeights); + + return true; } @@ -175,6 +325,7 @@ bool LaplacianPyramid::apply(const aliceVision::image::Image& int width = source.Width(); int height = source.Height(); + /* Convert mask to alpha layer */ image::Image mask_float(width, height); for(int i = 0; i < height; i++) @@ -296,7 +447,6 @@ bool LaplacianPyramid::apply(const aliceVision::image::Image& return true; } - bool LaplacianPyramid::merge(const aliceVision::image::Image& oimg, const aliceVision::image::Image& oweight, size_t level, size_t offset_x, size_t offset_y) @@ -351,11 +501,14 @@ bool LaplacianPyramid::merge(const aliceVision::image::Image& return false; } + + return true; } bool LaplacianPyramid::rebuild(CachedImage& output) { + // We first want to compute the final pixels mean for(int l = 0; l < _levels.size(); l++) diff --git a/src/aliceVision/panorama/warper.cpp b/src/aliceVision/panorama/warper.cpp index b4a36ca91b..7a5d2a0fe8 100644 --- a/src/aliceVision/panorama/warper.cpp +++ b/src/aliceVision/panorama/warper.cpp @@ -85,17 +85,31 @@ bool GaussianWarper::warp(const CoordinatesMap& map, const GaussianPyramidNoMask continue; } - if(i == _color.Height() - 1 || j == _color.Width() - 1 || !_mask(i + 1, j) || !_mask(i, j + 1)) + int next_j = j + 1; + int next_i = i + 1; + + if (j == _color.Width() - 1) + { + next_j = j - 1; + } + + if (i == _color.Height() - 1) + { + next_i = i - 1; + } + + if (!_mask(next_i, j) || !_mask(i, next_j)) { const Eigen::Vector2d& coord = coordinates(i, j); const image::RGBfColor pixel = sampler(mlsource[0], coord(1), coord(0)); _color(i, j) = pixel; + continue; } const Eigen::Vector2d& coord_mm = coordinates(i, j); - const Eigen::Vector2d& coord_mp = coordinates(i, j + 1); - const Eigen::Vector2d& coord_pm = coordinates(i + 1, j); + const Eigen::Vector2d& coord_mp = coordinates(i, next_j); + const Eigen::Vector2d& coord_pm = coordinates(next_i, j); double dxx = coord_pm(0) - coord_mm(0); double dxy = coord_mp(0) - coord_mm(0); diff --git a/src/software/pipeline/main_panoramaCompositing.cpp b/src/software/pipeline/main_panoramaCompositing.cpp index dd5168a1fd..2a56ac0b47 100644 --- a/src/software/pipeline/main_panoramaCompositing.cpp +++ b/src/software/pipeline/main_panoramaCompositing.cpp @@ -278,7 +278,7 @@ int aliceVision_main(int argc, char** argv) // Configure the cache manager memory cacheManager->setInCoreMaxObjectCount(1000); - LaplacianCompositer compositer(cacheManager, panoramaSize.first, panoramaSize.second, 1); + LaplacianCompositer compositer(cacheManager, panoramaSize.first, panoramaSize.second, 5); if (!compositer.initialize()) { diff --git a/src/software/pipeline/main_panoramaWarping.cpp b/src/software/pipeline/main_panoramaWarping.cpp index 05a3ac6a8e..66980b6494 100644 --- a/src/software/pipeline/main_panoramaWarping.cpp +++ b/src/software/pipeline/main_panoramaWarping.cpp @@ -403,7 +403,7 @@ int aliceVision_main(int argc, char** argv) } - #pragma omp parallel for + //#pragma omp parallel for { for (int i = 0; i < boxes.size(); i++) { @@ -412,6 +412,8 @@ int aliceVision_main(int argc, char** argv) int x = localBbox.left - snappedGlobalBbox.left; int y = localBbox.top - snappedGlobalBbox.top; + + // Prepare coordinates map CoordinatesMap map; if (!map.build(panoramaSize, camPose, *(intrinsic.get()), localBbox)) From 7699a19676a309d9c32451041eb101cdf1761f1e Mon Sep 17 00:00:00 2001 From: Fabien Servant Date: Wed, 7 Oct 2020 10:06:34 +0200 Subject: [PATCH 09/79] [panorama] wip --- src/aliceVision/panorama/cachedImage.hpp | 18 +++++++++++++----- src/aliceVision/panorama/compositer.hpp | 3 ++- src/aliceVision/panorama/distance.cpp | 2 -- src/aliceVision/panorama/laplacianPyramid.cpp | 11 ++++++++--- .../pipeline/main_panoramaCompositing.cpp | 19 ++++++++++++++++++- 5 files changed, 41 insertions(+), 12 deletions(-) diff --git a/src/aliceVision/panorama/cachedImage.hpp b/src/aliceVision/panorama/cachedImage.hpp index 3bf6034f33..292f34a045 100644 --- a/src/aliceVision/panorama/cachedImage.hpp +++ b/src/aliceVision/panorama/cachedImage.hpp @@ -249,6 +249,7 @@ class CachedImage { //ibb.top + i * tileSize --> snapedBb.top + delta + i * tileSize int ti = gridBb.top + i; + int oy = ti * _tileSize; int sy = inputBb.top - delta_y + i * _tileSize; std::vector& row = _tilesArray[ti]; @@ -256,6 +257,7 @@ class CachedImage for(int j = 0; j < gridBb.width; j++) { int tj = gridBb.left + j; + int ox = tj * _tileSize; int sx = inputBb.left - delta_x + j * _tileSize; image::CachedTile::smart_pointer ptr = row[tj]; @@ -275,8 +277,11 @@ class CachedImage { for(int x = 0; x < _tileSize; x++) { - if (sy + y < 0 || sy + y >= input.Height()) continue; - if (sx + x < 0 || sx + x >= input.Width()) continue; + if (sy + y < inputBb.top || sy + y > inputBb.getBottom()) continue; + if (sx + x < inputBb.left || sx + x > inputBb.getRight()) continue; + if (oy + y < outputBb.top || oy + y > outputBb.getBottom()) continue; + if (ox + x < outputBb.left || ox + x > outputBb.getRight()) continue; + data[y * _tileSize + x] = input(sy + y, sx + x); } } @@ -336,6 +341,7 @@ class CachedImage for(int i = 0; i < gridBb.height; i++) { int ti = gridBb.top + i; + int oy = ti * _tileSize; int sy = outputBb.top - delta_y + i * _tileSize; @@ -344,6 +350,7 @@ class CachedImage for(int j = 0; j < gridBb.width; j++) { int tj = gridBb.left + j; + int ox = tj * _tileSize; int sx = outputBb.left - delta_x + j * _tileSize; image::CachedTile::smart_pointer ptr = row[tj]; @@ -363,9 +370,10 @@ class CachedImage { for(int x = 0; x < _tileSize; x++) { - if (sy + y < 0 || sy + y >= output.Height()) continue; - if (sx + x < 0 || sx + x >= output.Width()) continue; - if (y < 0 || x < 0) continue; + if (sy + y < outputBb.top || sy + y > outputBb.getBottom()) continue; + if (sx + x < outputBb.left || sx + x > outputBb.getRight()) continue; + if (oy + y < inputBb.top || oy + y > inputBb.getBottom()) continue; + if (ox + x < inputBb.left || ox + x > inputBb.getRight()) continue; output(sy + y, sx + x) = data[y * _tileSize + x]; } diff --git a/src/aliceVision/panorama/compositer.hpp b/src/aliceVision/panorama/compositer.hpp index d41cc06336..98c61b311c 100644 --- a/src/aliceVision/panorama/compositer.hpp +++ b/src/aliceVision/panorama/compositer.hpp @@ -47,7 +47,8 @@ bool loopyCachedImageAssign(CachedImage & output, const aliceVision::image::I outputBb.left = left_1; inputBb.width = width1; outputBb.width = width1; - + + if (!output.assign(input, inputBb, outputBb)) { return false; diff --git a/src/aliceVision/panorama/distance.cpp b/src/aliceVision/panorama/distance.cpp index 02e7726143..bc89cf6938 100644 --- a/src/aliceVision/panorama/distance.cpp +++ b/src/aliceVision/panorama/distance.cpp @@ -1,7 +1,5 @@ #pragma once -#include "distance.hpp" - namespace aliceVision { diff --git a/src/aliceVision/panorama/laplacianPyramid.cpp b/src/aliceVision/panorama/laplacianPyramid.cpp index 8713180ff0..f534642f6e 100644 --- a/src/aliceVision/panorama/laplacianPyramid.cpp +++ b/src/aliceVision/panorama/laplacianPyramid.cpp @@ -492,17 +492,16 @@ bool LaplacianPyramid::merge(const aliceVision::image::Image& inputBb.width = extractedColor.Width(); inputBb.height = extractedColor.Height(); + + if (!loopyCachedImageAssign(img, extractedColor, extractBb, inputBb)) { return false; } - if (!loopyCachedImageAssign(weight, extractedWeight, extractBb, inputBb)) { return false; } - - return true; } @@ -532,6 +531,9 @@ bool LaplacianPyramid::rebuild(CachedImage& output) ); } + char filename[FILENAME_MAX]; + sprintf(filename, "/home/mmoc/file%d.exr", _levels.size() - 1); + _levels[_levels.size() - 1].writeImage(filename); removeNegativeValues(_levels[_levels.size() - 1]); @@ -611,6 +613,9 @@ bool LaplacianPyramid::rebuild(CachedImage& output) } removeNegativeValues(_levels[currentLevel]); + char filename[FILENAME_MAX]; + sprintf(filename, "/home/mmoc/file%d.exr", currentLevel); + _levels[currentLevel].writeImage(filename); } for(int i = 0; i < output.getTiles().size(); i++) diff --git a/src/software/pipeline/main_panoramaCompositing.cpp b/src/software/pipeline/main_panoramaCompositing.cpp index 2a56ac0b47..3560705c04 100644 --- a/src/software/pipeline/main_panoramaCompositing.cpp +++ b/src/software/pipeline/main_panoramaCompositing.cpp @@ -293,6 +293,10 @@ int aliceVision_main(int argc, char** argv) return EXIT_FAILURE; } + char filename[FILENAME_MAX]; + + labels.writeImage("/home/mmoc/labels.exr"); + /*if (!computeGCLabels(labels, cacheManager, sfmData, warpingFolder, panoramaSize)) { ALICEVISION_LOG_ERROR("Error computing graph cut labels"); @@ -324,6 +328,7 @@ int aliceVision_main(int argc, char** argv) } } + size_t pos = 0; for(const auto & view : viewOrderedByScale) { @@ -333,7 +338,8 @@ int aliceVision_main(int argc, char** argv) // skip unreconstructed views continue; } - + pos++; + // Load image and convert it to linear colorspace const std::string imagePath = (fs::path(warpingFolder) / (std::to_string(viewId) + ".exr")).string(); //ALICEVISION_LOG_INFO("Load image with path " << imagePath); @@ -348,8 +354,16 @@ int aliceVision_main(int argc, char** argv) const std::size_t contentW = metadata.find("AliceVision:contentW")->get_int(); const std::size_t contentH = metadata.find("AliceVision:contentH")->get_int(); + if (offsetY < 927) + { + continue; + } + + std::cout << imagePath << std::endl; + // Load mask const std::string maskPath = (fs::path(warpingFolder) / (std::to_string(viewId) + "_mask.exr")).string(); + //ALICEVISION_LOG_INFO("Load mask with path " << maskPath); image::Image mask; image::readImage(maskPath, mask, image::EImageColorSpace::NO_CONVERSION); @@ -373,6 +387,9 @@ int aliceVision_main(int argc, char** argv) return EXIT_FAILURE; } + sprintf(filename, "/home/mmoc/label%d.exr", pos); + image::writeImage(filename, seams, image::EImageColorSpace::NO_CONVERSION); + compositer.append(source, mask, seams, offsetX, offsetY, imageContent); } From 27084aa43252c85b7ecda4b32444b4155f102047 Mon Sep 17 00:00:00 2001 From: Fabien Servant Date: Wed, 7 Oct 2020 14:29:44 +0200 Subject: [PATCH 10/79] [panorama] working large scale compositing --- src/aliceVision/panorama/distance.cpp | 2 +- src/aliceVision/panorama/imageOps.cpp | 6 +++--- src/aliceVision/panorama/laplacianCompositer.hpp | 12 +++++------- src/aliceVision/panorama/laplacianPyramid.cpp | 16 +++++++--------- .../pipeline/main_panoramaCompositing.cpp | 14 +------------- 5 files changed, 17 insertions(+), 33 deletions(-) diff --git a/src/aliceVision/panorama/distance.cpp b/src/aliceVision/panorama/distance.cpp index bc89cf6938..df8bf44b7b 100644 --- a/src/aliceVision/panorama/distance.cpp +++ b/src/aliceVision/panorama/distance.cpp @@ -1,4 +1,4 @@ -#pragma once +#include "distance.hpp" namespace aliceVision { diff --git a/src/aliceVision/panorama/imageOps.cpp b/src/aliceVision/panorama/imageOps.cpp index 8b0bc75a75..293b417adb 100644 --- a/src/aliceVision/panorama/imageOps.cpp +++ b/src/aliceVision/panorama/imageOps.cpp @@ -11,9 +11,9 @@ void removeNegativeValues(CachedImage & img) image::RGBfColor rpix; image::RGBfColor ret = c; - rpix.r() = /*std::exp*/(c.r()); - rpix.g() = /*std::exp*/(c.g()); - rpix.b() = /*std::exp*/(c.b()); + rpix.r() = std::exp(c.r()); + rpix.g() = std::exp(c.g()); + rpix.b() = std::exp(c.b()); if (rpix.r() < 0.0) { diff --git a/src/aliceVision/panorama/laplacianCompositer.hpp b/src/aliceVision/panorama/laplacianCompositer.hpp index 5102e125b7..18e5237e58 100644 --- a/src/aliceVision/panorama/laplacianCompositer.hpp +++ b/src/aliceVision/panorama/laplacianCompositer.hpp @@ -100,9 +100,6 @@ class LaplacianCompositer : public Compositer size_t minsize = std::min(width, height); const float gaussian_filter_size = 5.0f; size_t optimal_scale = size_t(floor(std::log2(double(minsize) / gaussian_filter_size))); - - optimal_scale = 5; - return optimal_scale; } @@ -146,7 +143,7 @@ class LaplacianCompositer : public Compositer } /*To log space for hdr*/ - /*for(int i = 0; i < feathered.Height(); i++) + for(int i = 0; i < feathered.Height(); i++) { for(int j = 0; j < feathered.Width(); j++) { @@ -154,7 +151,7 @@ class LaplacianCompositer : public Compositer feathered(i, j).g() = std::log(std::max(1e-8f, feathered(i, j).g())); feathered(i, j).b() = std::log(std::max(1e-8f, feathered(i, j).b())); } - }*/ + } if (!_pyramidPanorama.apply(feathered, mask_pot, weights_pot, new_offset_x, new_offset_y)) { @@ -172,7 +169,7 @@ class LaplacianCompositer : public Compositer return false; } - /*_panorama.perPixelOperation( + _panorama.perPixelOperation( [](const image::RGBAfColor & a) -> image::RGBAfColor { image::RGBAfColor out; @@ -180,10 +177,11 @@ class LaplacianCompositer : public Compositer out.r() = std::exp(a.r()); out.g() = std::exp(a.g()); out.b() = std::exp(a.b()); + out.a() = a.a(); return out; } - );*/ + ); return true; } diff --git a/src/aliceVision/panorama/laplacianPyramid.cpp b/src/aliceVision/panorama/laplacianPyramid.cpp index f534642f6e..54f2f8da86 100644 --- a/src/aliceVision/panorama/laplacianPyramid.cpp +++ b/src/aliceVision/panorama/laplacianPyramid.cpp @@ -62,7 +62,7 @@ bool LaplacianPyramid::initialize(image::TileCacheManager::shared_ptr & cacheMan bool LaplacianPyramid::augment(image::TileCacheManager::shared_ptr & cacheManager, size_t newMaxLevels) { - return true; + //return true; if(newMaxLevels <= _levels.size()) { @@ -318,6 +318,8 @@ bool LaplacianPyramid::augment(image::TileCacheManager::shared_ptr & cacheManage return true; } + +static int pos = 0; bool LaplacianPyramid::apply(const aliceVision::image::Image& source, const aliceVision::image::Image& mask, const aliceVision::image::Image& weights, size_t offset_x, size_t offset_y) @@ -426,6 +428,7 @@ bool LaplacianPyramid::apply(const aliceVision::image::Image& } substract(current_color, current_color, buf2); + convolveGaussian5x5(buf_float, current_weights); downscale(next_weights, buf_float); @@ -443,7 +446,7 @@ bool LaplacianPyramid::apply(const aliceVision::image::Image& } merge(current_color, current_weights, _levels.size() - 1, offset_x, offset_y); - + pos++; return true; } @@ -531,9 +534,7 @@ bool LaplacianPyramid::rebuild(CachedImage& output) ); } - char filename[FILENAME_MAX]; - sprintf(filename, "/home/mmoc/file%d.exr", _levels.size() - 1); - _levels[_levels.size() - 1].writeImage(filename); + removeNegativeValues(_levels[_levels.size() - 1]); @@ -613,9 +614,6 @@ bool LaplacianPyramid::rebuild(CachedImage& output) } removeNegativeValues(_levels[currentLevel]); - char filename[FILENAME_MAX]; - sprintf(filename, "/home/mmoc/file%d.exr", currentLevel); - _levels[currentLevel].writeImage(filename); } for(int i = 0; i < output.getTiles().size(); i++) @@ -655,7 +653,7 @@ bool LaplacianPyramid::rebuild(CachedImage& output) if(ptrWeight[k] < 1e-6) { - ptrOutput[k].a() = 1.0f; + ptrOutput[k].a() = 0.0f; } else { diff --git a/src/software/pipeline/main_panoramaCompositing.cpp b/src/software/pipeline/main_panoramaCompositing.cpp index 3560705c04..5479b803d7 100644 --- a/src/software/pipeline/main_panoramaCompositing.cpp +++ b/src/software/pipeline/main_panoramaCompositing.cpp @@ -278,7 +278,7 @@ int aliceVision_main(int argc, char** argv) // Configure the cache manager memory cacheManager->setInCoreMaxObjectCount(1000); - LaplacianCompositer compositer(cacheManager, panoramaSize.first, panoramaSize.second, 5); + LaplacianCompositer compositer(cacheManager, panoramaSize.first, panoramaSize.second, 1); if (!compositer.initialize()) { @@ -293,9 +293,6 @@ int aliceVision_main(int argc, char** argv) return EXIT_FAILURE; } - char filename[FILENAME_MAX]; - - labels.writeImage("/home/mmoc/labels.exr"); /*if (!computeGCLabels(labels, cacheManager, sfmData, warpingFolder, panoramaSize)) { @@ -354,12 +351,6 @@ int aliceVision_main(int argc, char** argv) const std::size_t contentW = metadata.find("AliceVision:contentW")->get_int(); const std::size_t contentH = metadata.find("AliceVision:contentH")->get_int(); - if (offsetY < 927) - { - continue; - } - - std::cout << imagePath << std::endl; // Load mask const std::string maskPath = (fs::path(warpingFolder) / (std::to_string(viewId) + "_mask.exr")).string(); @@ -387,9 +378,6 @@ int aliceVision_main(int argc, char** argv) return EXIT_FAILURE; } - sprintf(filename, "/home/mmoc/label%d.exr", pos); - image::writeImage(filename, seams, image::EImageColorSpace::NO_CONVERSION); - compositer.append(source, mask, seams, offsetX, offsetY, imageContent); } From fb48f63fcf31ef9c4af74d7184a16f8d6a7cf544 Mon Sep 17 00:00:00 2001 From: Fabien Servant Date: Fri, 9 Oct 2020 15:11:39 +0200 Subject: [PATCH 11/79] [panorama] wip large graphcut --- src/aliceVision/panorama/cachedImage.hpp | 23 ++- src/aliceVision/panorama/graphcut.hpp | 165 +++++++++--------- src/aliceVision/panorama/laplacianPyramid.cpp | 2 + src/aliceVision/panorama/seams.cpp | 121 +++++++++---- src/aliceVision/panorama/seams.hpp | 21 ++- .../pipeline/main_panoramaCompositing.cpp | 38 ++-- 6 files changed, 224 insertions(+), 146 deletions(-) diff --git a/src/aliceVision/panorama/cachedImage.hpp b/src/aliceVision/panorama/cachedImage.hpp index 292f34a045..9e30b3887c 100644 --- a/src/aliceVision/panorama/cachedImage.hpp +++ b/src/aliceVision/panorama/cachedImage.hpp @@ -5,6 +5,7 @@ #include #include #include +#include namespace aliceVision { @@ -13,6 +14,10 @@ template class CachedImage { public: + using RowType = std::vector; + +public: + bool createImage(std::shared_ptr manager, size_t width, size_t height) { @@ -77,7 +82,7 @@ class CachedImage for(int i = 0; i < _tilesArray.size(); i++) { - std::vector& row = _tilesArray[i]; + RowType & row = _tilesArray[i]; for(int j = 0; j < _tilesArray[i].size(); j++) { @@ -112,8 +117,8 @@ class CachedImage for(int i = 0; i < _tilesArray.size(); i++) { - std::vector& row = _tilesArray[i]; - std::vector& rowOther = other.getTiles()[i]; + RowType& row = _tilesArray[i]; + RowType& rowOther = other.getTiles()[i]; for(int j = 0; j < _tilesArray[i].size(); j++) { @@ -158,8 +163,8 @@ class CachedImage for(int i = 0; i < _tilesArray.size(); i++) { - std::vector & row = _tilesArray[i]; - std::vector & rowSource = source._tilesArray[i]; + RowType & row = _tilesArray[i]; + RowType & rowSource = source._tilesArray[i]; for(int j = 0; j < _tilesArray[i].size(); j++) { @@ -252,7 +257,7 @@ class CachedImage int oy = ti * _tileSize; int sy = inputBb.top - delta_y + i * _tileSize; - std::vector& row = _tilesArray[ti]; + RowType & row = _tilesArray[ti]; for(int j = 0; j < gridBb.width; j++) { @@ -345,7 +350,7 @@ class CachedImage int sy = outputBb.top - delta_y + i * _tileSize; - std::vector& row = _tilesArray[ti]; + RowType & row = _tilesArray[ti]; for(int j = 0; j < gridBb.width; j++) { @@ -460,7 +465,7 @@ class CachedImage return true; } - std::vector>& getTiles() { return _tilesArray; } + std::vector& getTiles() { return _tilesArray; } int getWidth() const { return _width; } @@ -475,7 +480,7 @@ class CachedImage int _memoryHeight; int _tileSize; - std::vector> _tilesArray; + std::vector _tilesArray; }; template <> diff --git a/src/aliceVision/panorama/graphcut.hpp b/src/aliceVision/panorama/graphcut.hpp index b366442ea2..cde952cb61 100644 --- a/src/aliceVision/panorama/graphcut.hpp +++ b/src/aliceVision/panorama/graphcut.hpp @@ -224,15 +224,23 @@ class GraphcutSeams virtual ~GraphcutSeams() = default; - void setOriginalLabels(CachedImage & existing_labels) - { - - _labels.deepCopy(existing_labels); - _original_labels.deepCopy(existing_labels); + bool setOriginalLabels(CachedImage & existing_labels) + { + if (!_labels.deepCopy(existing_labels)) + { + return false; + } + + if (!_original_labels.deepCopy(existing_labels)) + { + return false; + } /*image::Image seams(_labels.Width(), _labels.Height()); computeSeamsMap(seams, _labels); computeDistanceMap(_distancesSeams, seams);*/ + + return true; } bool initialize(image::TileCacheManager::shared_ptr & cacheManager) @@ -247,20 +255,16 @@ class GraphcutSeams return false; } - if(!_distancesSeams.createImage(cacheManager, _outputWidth, _outputHeight)) + /*if(!_distancesSeams.createImage(cacheManager, _outputWidth, _outputHeight)) { return false; } - if(!_distancesSeams.perPixelOperation( - [](int) -> int - { - return 0; - }) - ) + if(!_distancesSeams.fill(0.0f)) { return false; - } + } */ + return true; } @@ -268,7 +272,7 @@ class GraphcutSeams bool append(const aliceVision::image::Image& input, const aliceVision::image::Image& inputMask, IndexT currentIndex, size_t offset_x, size_t offset_y) { - if(inputMask.size() != input.size()) + /* if(inputMask.size() != input.size()) { return false; } @@ -280,22 +284,18 @@ class GraphcutSeams rect.width = input.Width() + 1; rect.height = input.Height() + 1; - /*Extend rect for borders*/ - rect.left = std::max(0, rect.left - 3); - rect.top = std::max(0, rect.top - 3); - rect.width = rect.width + 6; - rect.height = rect.height + 6; - if(rect.top + rect.height > _owners.Height()) - { - rect.height = _owners.Height() - rect.top; - } + //Extend rect for borders + rect.dilate(3); + rect.clampLeft(); + rect.clampTop(); + rect.clampBottom(_owners.Height() - 1) _rects[currentIndex] = rect; - /* - _owners will get for each pixel of the panorama a list of pixels - in the sources which may have seen this point. - */ + + //_owners will get for each pixel of the panorama a list of pixels + //in the sources which may have seen this point. + for(int i = 0; i < input.Height(); i++) { @@ -317,7 +317,7 @@ class GraphcutSeams info.first = currentIndex; info.second = input(i, j); - /* If too far away from seam, do not add a contender */ + // If too far away from seam, do not add a contender int dist = _distancesSeams(di, dj); if(dist > _maximal_distance_change + 10) { @@ -326,20 +326,23 @@ class GraphcutSeams _owners(di, dj).push_back(info); } - } + }*/ return true; } - void setMaximalDistance(int dist) { _maximal_distance_change = dist; } + void setMaximalDistance(int dist) + { + _maximal_distance_change = dist; + } bool process() { - for(int i = 0; i < 10; i++) + /*for(int i = 0; i < 10; i++) { - /*For each possible label, try to extends its domination on the label's world */ + // For each possible label, try to extends its domination on the label's world bool change = false; for(auto & info : _rects) @@ -381,7 +384,7 @@ class GraphcutSeams { break; } - } + }*/ return true; } @@ -393,7 +396,7 @@ class GraphcutSeams double cost = 0.0; - for(int i = 0; i < rect.height - 1; i++) + /*for(int i = 0; i < rect.height - 1; i++) { int y = rect.top + i; @@ -528,14 +531,14 @@ class GraphcutSeams cost += (XColorLC - XColorLX).norm(); cost += (YColorLC - YColorLY).norm(); } - } + }*/ return cost; } bool alphaExpansion(IndexT currentLabel) { - + /* BoundingBox rect = _rects[currentLabel]; image::Image mask(rect.width, rect.height, true, 0); @@ -543,7 +546,7 @@ class GraphcutSeams image::Image color_label(rect.width, rect.height, true, image::RGBfColor(0.0f, 0.0f, 0.0f)); image::Image color_other(rect.width, rect.height, true, image::RGBfColor(0.0f, 0.0f, 0.0f)); - /*Compute distance map to seams*/ + // Compute distance map to seams image::Image distanceMap(rect.width, rect.height); { image::Image binarizedWorld(rect.width, rect.height); @@ -585,15 +588,15 @@ class GraphcutSeams } } - /* - A warped input has valid pixels only in some parts of the final image. - Rect is the bounding box of these valid pixels. - Let's build a mask : - - 0 if the pixel is not viewed by anyone - - 1 if the pixel is viewed by the current label alpha - - 2 if the pixel is viewed by *another* label and this label is marked as current valid label - - 3 if the pixel is 1 + 2 : the pixel is not selected as alpha territory, but alpha is looking at it - */ + + //A warped input has valid pixels only in some parts of the final image. + //Rect is the bounding box of these valid pixels. + //Let's build a mask : + // - 0 if the pixel is not viewed by anyone + // - 1 if the pixel is viewed by the current label alpha + // - 2 if the pixel is viewed by *another* label and this label is marked as current valid label + // - 3 if the pixel is 1 + 2 : the pixel is not selected as alpha territory, but alpha is looking at it + for(int i = 0; i < rect.height; i++) { @@ -616,7 +619,7 @@ class GraphcutSeams int dist = distanceMap(i, j); - /* Loop over observations */ + // Loop over observations for(int l = 0; l < infos.size(); l++) { @@ -652,9 +655,7 @@ class GraphcutSeams } } - /* - If the pixel may be a new kingdom for alpha ! - */ + // If the pixel may be a new kingdom for alpha if(mask(i, j) == 1) { color_label(i, j) = currentColor; @@ -673,11 +674,9 @@ class GraphcutSeams } } - /* - The rectangle is a grid. - However we want to ignore a lot of pixel. - Let's create an index per valid pixels for graph cut reference - */ + // The rectangle is a grid. + // However we want to ignore a lot of pixel. + // Let's create an index per valid pixels for graph cut reference int count = 0; for(int i = 0; i < rect.height; i++) { @@ -693,7 +692,7 @@ class GraphcutSeams } } - /*Create graph*/ + //Create graph MaxFlow_AdjList gc(count); size_t countValid = 0; @@ -702,13 +701,13 @@ class GraphcutSeams for(int j = 0; j < rect.width; j++) { - /* If this pixel is not valid, ignore */ + // If this pixel is not valid, ignore if(mask(i, j) == 0) { continue; } - /* Get this pixel ID */ + // Get this pixel ID int node_id = ids(i, j); int im1 = std::max(i - 1, 0); @@ -719,45 +718,40 @@ class GraphcutSeams if(mask(i, j) == 1) { - /* Only add nodes close to borders */ + // Only add nodes close to borders if(mask(im1, jm1) == 1 && mask(im1, j) == 1 && mask(im1, jp1) == 1 && mask(i, jm1) == 1 && mask(i, jp1) == 1 && mask(ip1, jm1) == 1 && mask(ip1, j) == 1 && mask(ip1, jp1) == 1) { continue; } - /* - This pixel is only seen by alpha. - Enforce its domination by stating that removing this pixel - from alpha territoy is infinitly costly (impossible). - */ + + //This pixel is only seen by alpha. + //Enforce its domination by stating that removing this pixel + //from alpha territoy is infinitly costly (impossible). gc.addNodeToSource(node_id, 100000); } else if(mask(i, j) == 2) { - /* Only add nodes close to borders */ + // Only add nodes close to borders if(mask(im1, jm1) == 2 && mask(im1, j) == 2 && mask(im1, jp1) == 2 && mask(i, jm1) == 2 && mask(i, jp1) == 2 && mask(ip1, jm1) == 2 && mask(ip1, j) == 2 && mask(ip1, jp1) == 2) { continue; } - /* - This pixel is only seen by an ennemy. - Enforce its domination by stating that removing this pixel - from ennemy territory is infinitly costly (impossible). - */ + //This pixel is only seen by an ennemy. + //Enforce its domination by stating that removing this pixel + //from ennemy territory is infinitly costly (impossible). gc.addNodeToSink(node_id, 100000); } else if(mask(i, j) == 3) { - /* - This pixel is seen by both alpha and enemies but is owned by ennemy. - Make sure that changing node owner will have no direct cost. - Connect it to both alpha and ennemy for the moment - (Graph cut will not allow a pixel to have both owners at the end). - */ + // This pixel is seen by both alpha and enemies but is owned by ennemy. + // Make sure that changing node owner will have no direct cost. + // Connect it to both alpha and ennemy for the moment + // (Graph cut will not allow a pixel to have both owners at the end). gc.addNodeToSource(node_id, 0); gc.addNodeToSink(node_id, 0); countValid++; @@ -767,17 +761,16 @@ class GraphcutSeams if(countValid == 0) { - /* We have no possibility for territory expansion */ - /* let's exit */ + // We have no possibility for territory expansion + // let's exit return true; } - /* - Loop over alpha bounding box. - Let's define the transition cost. - When two neighboor pixels have different labels, there is a seam (border) cost. - Graph cut will try to make sure the territory will have a minimal border cost - */ + // Loop over alpha bounding box. + // Let's define the transition cost. + // When two neighboor pixels have different labels, there is a seam (border) cost. + // Graph cut will try to make sure the territory will have a minimal border cost + for(int i = 0; i < rect.height; i++) { for(int j = 0; j < rect.width; j++) @@ -790,11 +783,11 @@ class GraphcutSeams int node_id = ids(i, j); - /* Make sure it is possible to estimate this horizontal border */ + // Make sure it is possible to estimate this horizontal border if(i < mask.Height() - 1) { - /* Make sure the other pixel is owned by someone */ + // Make sure the other pixel is owned by someone if(mask(i + 1, j)) { diff --git a/src/aliceVision/panorama/laplacianPyramid.cpp b/src/aliceVision/panorama/laplacianPyramid.cpp index 54f2f8da86..b8d21b7ab3 100644 --- a/src/aliceVision/panorama/laplacianPyramid.cpp +++ b/src/aliceVision/panorama/laplacianPyramid.cpp @@ -63,6 +63,8 @@ bool LaplacianPyramid::initialize(image::TileCacheManager::shared_ptr & cacheMan bool LaplacianPyramid::augment(image::TileCacheManager::shared_ptr & cacheManager, size_t newMaxLevels) { //return true; + + ALICEVISION_LOG_INFO("augment number of levels to " << newMaxLevels); if(newMaxLevels <= _levels.size()) { diff --git a/src/aliceVision/panorama/seams.cpp b/src/aliceVision/panorama/seams.cpp index ddd52b2acf..72d6efac1b 100644 --- a/src/aliceVision/panorama/seams.cpp +++ b/src/aliceVision/panorama/seams.cpp @@ -143,11 +143,7 @@ bool WTASeams::initialize(image::TileCacheManager::shared_ptr & cacheManager) return false; } - if(!_weights.perPixelOperation( - [](float ) -> float - { - return 0.0f; - })) + if(!_weights.fill(0.0f)) { return false; } @@ -157,11 +153,7 @@ bool WTASeams::initialize(image::TileCacheManager::shared_ptr & cacheManager) return false; } - if(!_labels.perPixelOperation( - [](IndexT ) -> IndexT - { - return UndefinedIndexT; - })) + if(!_labels.fill(UndefinedIndexT)) { return false; } @@ -232,42 +224,93 @@ bool WTASeams::append(const aliceVision::image::Image& inputMask, return true; } -void HierarchicalGraphcutSeams::setOriginalLabels(CachedImage& labels) +bool HierarchicalGraphcutSeams::setOriginalLabels(CachedImage& labels) { + if (_levelOfInterest == 0) + { + return _graphcut->setOriginalLabels(labels); + } + + int scale = pow(2, _levelOfInterest); + int nw = _outputWidth / scale; + int nh = _outputHeight / scale; - /* - First of all, Propagate label to all levels - */ - /*image::Image current_label = labels; - - for(int l = 1; l <= _levelOfInterest; l++) + CachedImage smallLabels; + if(!smallLabels.createImage(_cacheManager, nw, nh)) { + return false; + } - aliceVision::image::Image next_label(current_label.Width() / 2, current_label.Height() / 2); + int processingSize = 256; + int largeSize = 256 * scale; - for(int i = 0; i < next_label.Height(); i++) + for (int i = 0; i < smallLabels.getHeight(); i+= processingSize) + { + for (int j = 0; j < smallLabels.getWidth(); j+= processingSize) { - int di = i * 2; + BoundingBox smallBb; + smallBb.left = j; + smallBb.top = i; + smallBb.width = processingSize; + smallBb.height = processingSize; + smallBb.clampRight(smallLabels.getWidth() - 1); + smallBb.clampBottom(smallLabels.getHeight() - 1); + + BoundingBox smallInputBb; + smallInputBb.left = 0; + smallInputBb.top = 0; + smallInputBb.width = smallBb.width; + smallInputBb.height = smallBb.height; + + + image::Image smallView(smallBb.width, smallBb.height); + + if (!smallLabels.extract(smallView, smallInputBb, smallBb)) + { + return false; + } + + BoundingBox largeBb; + largeBb.left = smallBb.left * scale; + largeBb.top = smallBb.top * scale; + largeBb.width = smallBb.width * scale; + largeBb.height = smallBb.height * scale; + + BoundingBox largeInputBb; + largeInputBb.left = 0; + largeInputBb.top = 0; + largeInputBb.width = largeBb.width; + largeInputBb.height = largeBb.height; - for(int j = 0; j < next_label.Width(); j++) + image::Image largeView(largeBb.width, largeBb.height); + if (!labels.extract(largeView, largeInputBb, largeBb)) { - int dj = j * 2; + return false; + } - next_label(i, j) = current_label(di, dj); + for (int y = 0; y < smallBb.height; y++) + { + for (int x = 0; x < smallBb.width; x++) + { + smallView(y, x) = largeView(y * scale, x * scale); + } } - } - current_label = next_label; + if (!smallLabels.assign(smallView, smallInputBb, smallBb)) + { + return false; + } + } } - _graphcut->setOriginalLabels(current_label);*/ + return _graphcut->setOriginalLabels(smallLabels); } bool HierarchicalGraphcutSeams::append(const aliceVision::image::Image& input, const aliceVision::image::Image& inputMask, IndexT currentIndex, size_t offset_x, size_t offset_y) { - /*image::Image current_color = input; + image::Image current_color = input; image::Image current_mask = inputMask; for(int l = 1; l <= _levelOfInterest; l++) @@ -306,9 +349,12 @@ bool HierarchicalGraphcutSeams::append(const aliceVision::image::Imageappend(current_color, current_mask, currentIndex, offset_x, offset_y);*/ - return true; + char filename[FILENAME_MAX]; + sprintf(filename, "/home/mmoc/test%d.exr", int(offset_x)); + image::writeImage(filename, current_color, image::EImageColorSpace::NO_CONVERSION); + + return _graphcut->append(current_color, current_mask, currentIndex, offset_x, offset_y); } bool HierarchicalGraphcutSeams::process() @@ -353,8 +399,23 @@ bool HierarchicalGraphcutSeams::process() return true; } -bool HierarchicalGraphcutSeams::initialize(image::TileCacheManager::shared_ptr & cacheManager) +bool HierarchicalGraphcutSeams::initialize() { + if (!_graphcut->initialize(_cacheManager)) + { + return false; + } + + if(!_labels.createImage(_cacheManager, _outputWidth, _outputHeight)) + { + return false; + } + + if(!_labels.fill(UndefinedIndexT)) + { + return false; + } + return true; } diff --git a/src/aliceVision/panorama/seams.hpp b/src/aliceVision/panorama/seams.hpp index e5e762134e..7bd0e51933 100644 --- a/src/aliceVision/panorama/seams.hpp +++ b/src/aliceVision/panorama/seams.hpp @@ -4,7 +4,7 @@ #include #include "cachedImage.hpp" -//#include "graphcut.hpp" +#include "graphcut.hpp" namespace aliceVision { @@ -48,22 +48,27 @@ class WTASeams class HierarchicalGraphcutSeams { public: - HierarchicalGraphcutSeams(size_t outputWidth, size_t outputHeight, size_t levelOfInterest) - : _outputWidth(outputWidth) + HierarchicalGraphcutSeams(image::TileCacheManager::shared_ptr cacheManager, size_t outputWidth, size_t outputHeight, size_t levelOfInterest) + : _cacheManager(cacheManager) + , _outputWidth(outputWidth) , _outputHeight(outputHeight) , _levelOfInterest(levelOfInterest) { + double scale = 1.0 / pow(2.0, levelOfInterest); + size_t width = size_t(floor(double(outputWidth) * scale)); + size_t height = size_t(floor(double(outputHeight) * scale)); + _graphcut = std::unique_ptr(new GraphcutSeams(width, height)); } virtual ~HierarchicalGraphcutSeams() = default; - bool initialize(image::TileCacheManager::shared_ptr & cacheManager); + bool initialize(); - void setOriginalLabels(CachedImage& labels); + bool setOriginalLabels(CachedImage& labels); void setMaximalDistance(int distance) { - //_graphcut->setMaximalDistance(distance); + _graphcut->setMaximalDistance(distance); } virtual bool append(const aliceVision::image::Image& input, @@ -78,8 +83,10 @@ class HierarchicalGraphcutSeams } private: - //std::unique_ptr _graphcut; + std::unique_ptr _graphcut; + image::TileCacheManager::shared_ptr _cacheManager; CachedImage _labels; + size_t _levelOfInterest; size_t _outputWidth; size_t _outputHeight; diff --git a/src/software/pipeline/main_panoramaCompositing.cpp b/src/software/pipeline/main_panoramaCompositing.cpp index 5479b803d7..3aee05a0f1 100644 --- a/src/software/pipeline/main_panoramaCompositing.cpp +++ b/src/software/pipeline/main_panoramaCompositing.cpp @@ -89,31 +89,36 @@ bool computeWTALabels(CachedImage & labels, image::TileCacheManager::sha bool computeGCLabels(CachedImage & labels, image::TileCacheManager::shared_ptr& cacheManager, const sfmData::SfMData& sfmData, const std::string & inputPath, std::pair & panoramaSize) { - //Compute finest level possible for graph cut + //Compute coarsest level possible for graph cut int initial_level = 0; - int max_width_for_graphcut = 5000; - double ratio = double(panoramaSize.first) / double(max_width_for_graphcut); + int min_width_for_graphcut = 1000; + double ratio = double(panoramaSize.first) / double(min_width_for_graphcut); if (ratio > 1.0) { initial_level = int(ceil(log2(ratio))); } - for (int l = initial_level; l>= 0; l--) + for (int l = initial_level; l>= initial_level; l--) { - HierarchicalGraphcutSeams seams(panoramaSize.first, panoramaSize.second, l); + + HierarchicalGraphcutSeams seams(cacheManager, panoramaSize.first, panoramaSize.second, l); - if (!seams.initialize(cacheManager)) + + if (!seams.initialize()) + { + return false; + } + + if (!seams.setOriginalLabels(labels)) { return false; } - seams.setOriginalLabels(labels); - if (l != initial_level) { seams.setMaximalDistance(100); } - for (const auto& viewIt : sfmData.getViews()) + /*for (const auto& viewIt : sfmData.getViews()) { if(!sfmData.isPoseAndIntrinsicDefined(viewIt.second.get())) { @@ -142,12 +147,13 @@ bool computeGCLabels(CachedImage & labels, image::TileCacheManager::shar seams.append(colors, mask, viewIt.first, offsetX, offsetY); } - if (seams.process()) + if (!seams.process()) { - labels = seams.getLabels(); + return false; } - } + labels = seams.getLabels();*/ + } return true; } @@ -294,11 +300,15 @@ int aliceVision_main(int argc, char** argv) } - /*if (!computeGCLabels(labels, cacheManager, sfmData, warpingFolder, panoramaSize)) + if (!computeGCLabels(labels, cacheManager, sfmData, warpingFolder, panoramaSize)) { ALICEVISION_LOG_ERROR("Error computing graph cut labels"); return EXIT_FAILURE; - }*/ + } + + labels.writeImage("/home/mmoc/labels.exr"); + + return EXIT_SUCCESS; //Get a list of views ordered by their image scale From 4f81779d43155c8cd1c421005f29eb10799c5e02 Mon Sep 17 00:00:00 2001 From: Fabien Servant Date: Mon, 12 Oct 2020 12:08:52 +0200 Subject: [PATCH 12/79] [panorama] graphcut wip --- src/aliceVision/panorama/CMakeLists.txt | 15 +++ src/aliceVision/panorama/boundingBox.hpp | 94 +++++++++++++++---- src/aliceVision/panorama/boundingBoxMap.cpp | 52 ++++++++++ src/aliceVision/panorama/boundingBoxMap.hpp | 24 +++++ src/aliceVision/panorama/graphcut.hpp | 60 ++++++------ src/aliceVision/panorama/imageOps.cpp | 48 ++++++++++ src/aliceVision/panorama/imageOps.hpp | 2 + src/aliceVision/panorama/seams.cpp | 81 +++++++++------- src/aliceVision/panorama/seams.hpp | 9 +- .../pipeline/main_panoramaCompositing.cpp | 66 ++++++++++--- .../pipeline/main_panoramaWarping.cpp | 2 +- 11 files changed, 356 insertions(+), 97 deletions(-) create mode 100644 src/aliceVision/panorama/boundingBoxMap.cpp create mode 100644 src/aliceVision/panorama/boundingBoxMap.hpp diff --git a/src/aliceVision/panorama/CMakeLists.txt b/src/aliceVision/panorama/CMakeLists.txt index 5d48cfaa4a..54a1fb80af 100644 --- a/src/aliceVision/panorama/CMakeLists.txt +++ b/src/aliceVision/panorama/CMakeLists.txt @@ -1,6 +1,20 @@ # Headers set(panorama_files_headers + alphaCompositer.hpp boundingBox.hpp + boundingBoxMap.hpp + compositer.hpp + coordinatesMap.hpp + distance.hpp + feathering.hpp + gaussian.hpp + graphcut.hpp + imageOps.hpp + laplacianCompositer.hpp + laplacianPyramid.hpp + remapBbox.hpp + seams.hpp + sphericalMapping.hpp warper.hpp ) @@ -17,6 +31,7 @@ set(panorama_files_sources seams.cpp imageOps.cpp cachedImage.cpp + boundingBoxMap.cpp ) alicevision_add_library(aliceVision_panorama diff --git a/src/aliceVision/panorama/boundingBox.hpp b/src/aliceVision/panorama/boundingBox.hpp index b542e89e5a..4d5b1e877d 100644 --- a/src/aliceVision/panorama/boundingBox.hpp +++ b/src/aliceVision/panorama/boundingBox.hpp @@ -1,5 +1,12 @@ #pragma once +#include +#include +#include + +namespace aliceVision +{ + struct BoundingBox { @@ -24,9 +31,17 @@ struct BoundingBox { } - int getRight() const { return left + width - 1; } + constexpr int getRight() const { + return left + width - 1; + } + + constexpr int getBottom() const { + return top + height - 1; + } - int getBottom() const { return top + height - 1; } + constexpr bool isEmpty() const { + return (width <= 0 || height <= 0); + } void snapToGrid(uint32_t gridSize) { @@ -34,10 +49,10 @@ struct BoundingBox int right = getRight(); int bottom = getBottom(); - int leftBounded = int(floor(double(left) / double(gridSize))) * int(gridSize); - int topBounded = int(floor(double(top) / double(gridSize))) * int(gridSize); - int widthBounded = int(ceil(double(right - leftBounded + 1) / double(gridSize))) * int(gridSize); - int heightBounded = int(ceil(double(bottom - topBounded + 1) / double(gridSize))) * int(gridSize); + int leftBounded = int(std::floor(double(left) / double(gridSize))) * int(gridSize); + int topBounded = int(std::floor(double(top) / double(gridSize))) * int(gridSize); + int widthBounded = int(std::ceil(double(right - leftBounded + 1) / double(gridSize))) * int(gridSize); + int heightBounded = int(std::ceil(double(bottom - topBounded + 1) / double(gridSize))) * int(gridSize); left = leftBounded; top = topBounded; @@ -45,16 +60,17 @@ struct BoundingBox height = heightBounded; } - void unionWith(const BoundingBox& other) + BoundingBox unionWith(const BoundingBox& other) const { + BoundingBox ret; - if(left < 0 && top < 0) + if (left < 0 && top < 0) { - left = other.left; - top = other.top; - width = other.width; - height = other.height; - return; + ret.left = other.left; + ret.top = other.top; + ret.width = other.width; + ret.height = other.height; + return ret; } int rt = getRight(); @@ -62,14 +78,32 @@ struct BoundingBox int bt = getBottom(); int bo = other.getBottom(); - left = std::min(left, other.left); - top = std::min(top, other.top); + ret.left = std::min(left, other.left); + ret.top = std::min(top, other.top); int maxr = std::max(rt, ro); int maxb = std::max(bt, bo); - width = maxr - left + 1; - height = maxb - top + 1; + ret.width = maxr - left + 1; + ret.height = maxb - top + 1; + + return ret; + } + + BoundingBox intersectionWith(const BoundingBox& other) const + { + BoundingBox intersection; + + intersection.left = std::max(left, other.left); + intersection.top = std::max(top, other.top); + + int right = std::min(getRight(), other.getRight()); + int bottom = std::min(getBottom(), other.getBottom()); + + intersection.width = std::max(0, right - intersection.left + 1); + intersection.height = std::max(0, bottom - intersection.top + 1); + + return intersection; } bool isInside(const BoundingBox& other) const @@ -143,4 +177,30 @@ struct BoundingBox return b; } + + BoundingBox multiply(int factor) + { + BoundingBox b; + + b.left = left * factor; + b.top = top * factor; + b.width = width * factor; + b.height = height * factor; + + return b; + } + + BoundingBox divide(int factor) + { + BoundingBox b; + + b.left = left / factor; + b.top = top / factor; + b.width = width / factor; + b.height = height / factor; + + return b; + } }; + +} \ No newline at end of file diff --git a/src/aliceVision/panorama/boundingBoxMap.cpp b/src/aliceVision/panorama/boundingBoxMap.cpp new file mode 100644 index 0000000000..0e785d7e8f --- /dev/null +++ b/src/aliceVision/panorama/boundingBoxMap.cpp @@ -0,0 +1,52 @@ +#include "boundingBoxMap.hpp" + +#include + +namespace aliceVision +{ + +bool BoundingBoxMap::append(IndexT index, const BoundingBox & box) +{ + _map[index] = box; + + return true; +} + +bool BoundingBoxMap::getIntersectionList(std::map & intersections, IndexT reference) const +{ + auto it = _map.find(reference); + + if (it == _map.end()) + { + return false; + } + + intersections.clear(); + + BoundingBox refBB = it->second; + if (refBB.isEmpty()) + { + return false; + } + + for (auto item : _map) + { + if (item.first == reference) + { + continue; + } + + BoundingBox intersection = refBB.intersectionWith(item.second); + if (intersection.isEmpty()) + { + continue; + } + + intersections[item.first] = intersection; + } + + return true; +} + + +} \ No newline at end of file diff --git a/src/aliceVision/panorama/boundingBoxMap.hpp b/src/aliceVision/panorama/boundingBoxMap.hpp new file mode 100644 index 0000000000..ca56676b84 --- /dev/null +++ b/src/aliceVision/panorama/boundingBoxMap.hpp @@ -0,0 +1,24 @@ +#pragma once + +#include "boundingBox.hpp" + +#include +#include + +namespace aliceVision +{ + +class BoundingBoxMap +{ +public: + BoundingBoxMap() = default; + + bool append(IndexT index, const BoundingBox & box); + + bool getIntersectionList(std::map & intersections, IndexT reference) const; + +private: + std::map _map; +}; + +} \ No newline at end of file diff --git a/src/aliceVision/panorama/graphcut.hpp b/src/aliceVision/panorama/graphcut.hpp index cde952cb61..45d89537aa 100644 --- a/src/aliceVision/panorama/graphcut.hpp +++ b/src/aliceVision/panorama/graphcut.hpp @@ -212,6 +212,14 @@ bool computeSeamsMap(image::Image& seams, const image::Image color; + CachedImage mask; + }; + using PixelInfo = std::pair; using ImageOwners = image::Image>; @@ -231,15 +239,6 @@ class GraphcutSeams return false; } - if (!_original_labels.deepCopy(existing_labels)) - { - return false; - } - - /*image::Image seams(_labels.Width(), _labels.Height()); - computeSeamsMap(seams, _labels); - computeDistanceMap(_distancesSeams, seams);*/ - return true; } @@ -250,32 +249,34 @@ class GraphcutSeams return false; } - if(!_original_labels.createImage(cacheManager, _outputWidth, _outputHeight)) - { - return false; - } + return true; + } - /*if(!_distancesSeams.createImage(cacheManager, _outputWidth, _outputHeight)) + bool append(const CachedImage& input, const CachedImage& inputMask, IndexT currentIndex, size_t offset_x, size_t offset_y) + { + + if(inputMask.getWidth() != input.getWidth()) { return false; } - if(!_distancesSeams.fill(0.0f)) + if(inputMask.getHeight() != input.getHeight()) { return false; - } */ + } + InputData data; + data.id = currentIndex; + data.color = input; + data.mask = inputMask; + data.rect.width = input.getWidth(); + data.rect.height = input.getHeight(); + data.rect.left = offset_x; + data.rect.top = offset_y; - return true; - } - - bool append(const aliceVision::image::Image& input, const aliceVision::image::Image& inputMask, IndexT currentIndex, size_t offset_x, size_t offset_y) - { + _inputs[currentIndex] = data; - /* if(inputMask.size() != input.size()) - { - return false; - } + /* BoundingBox rect; @@ -319,7 +320,7 @@ class GraphcutSeams // If too far away from seam, do not add a contender int dist = _distancesSeams(di, dj); - if(dist > _maximal_distance_change + 10) + if (dist > _maximal_distance_change + 10) { continue; } @@ -392,7 +393,7 @@ class GraphcutSeams double cost(IndexT currentLabel) { - BoundingBox rect = _rects[currentLabel]; + //BoundingBox rect = _rects[currentLabel]; double cost = 0.0; @@ -874,14 +875,13 @@ class GraphcutSeams private: - std::map _rects; + std::map _inputs; + int _outputWidth; int _outputHeight; size_t _maximal_distance_change; CachedImage _labels; - CachedImage _original_labels; - CachedImage _distancesSeams; ImageOwners _owners; }; diff --git a/src/aliceVision/panorama/imageOps.cpp b/src/aliceVision/panorama/imageOps.cpp index 293b417adb..56efd84480 100644 --- a/src/aliceVision/panorama/imageOps.cpp +++ b/src/aliceVision/panorama/imageOps.cpp @@ -1,4 +1,5 @@ #include "imageOps.hpp" +#include "gaussian.hpp" namespace aliceVision { @@ -35,4 +36,51 @@ void removeNegativeValues(CachedImage & img) ); } +bool downscaleByPowerOfTwo(image::Image & output, image::Image & outputMask, const image::Image & input, const image::Image & inputMask, const int timesDividedBy2) +{ + image::Image currentColor = input; + image::Image currentMask = inputMask; + + for(int l = 1; l <= timesDividedBy2; l++) + { + + aliceVision::image::Image buf(currentColor.Width(), currentColor.Height()); + aliceVision::image::Image nextColor(currentColor.Width() / 2, currentColor.Height() / 2); + aliceVision::image::Image nextMask(currentColor.Width() / 2, currentColor.Height() / 2); + + //Convolve + divide + convolveGaussian5x5(buf, currentColor); + downscale(nextColor, buf); + + //Just nearest neighboor divide for mask + for(int i = 0; i < nextMask.Height(); i++) + { + int di = i * 2; + + for(int j = 0; j < nextMask.Width(); j++) + { + int dj = j * 2; + + if(currentMask(di, dj) && currentMask(di, dj + 1) + && currentMask(di + 1, dj) && currentMask(di + 1, dj + 1)) + { + nextMask(i, j) = 255; + } + else + { + nextMask(i, j) = 0; + } + } + } + + currentColor = nextColor; + currentMask = nextMask; + } + + output = currentColor; + outputMask = currentMask; + + return true; +} + } // namespace aliceVision \ No newline at end of file diff --git a/src/aliceVision/panorama/imageOps.hpp b/src/aliceVision/panorama/imageOps.hpp index 69333c1897..28e2320bee 100644 --- a/src/aliceVision/panorama/imageOps.hpp +++ b/src/aliceVision/panorama/imageOps.hpp @@ -112,6 +112,8 @@ bool addition(aliceVision::image::Image& AplusB, const aliceVision::image::Im return true; } +bool downscaleByPowerOfTwo(image::Image & output, image::Image & outputMask, const image::Image & input, const image::Image & inputMask, const int timesDividedBy2); + void removeNegativeValues(CachedImage& img); } // namespace aliceVision \ No newline at end of file diff --git a/src/aliceVision/panorama/seams.cpp b/src/aliceVision/panorama/seams.cpp index 72d6efac1b..1d3b2e90d5 100644 --- a/src/aliceVision/panorama/seams.cpp +++ b/src/aliceVision/panorama/seams.cpp @@ -308,58 +308,67 @@ bool HierarchicalGraphcutSeams::setOriginalLabels(CachedImage& labels) bool HierarchicalGraphcutSeams::append(const aliceVision::image::Image& input, const aliceVision::image::Image& inputMask, IndexT currentIndex, - size_t offset_x, size_t offset_y) + size_t offsetX, size_t offsetY) { - image::Image current_color = input; - image::Image current_mask = inputMask; + image::Image resizedColor; + image::Image resizedMask; - for(int l = 1; l <= _levelOfInterest; l++) + int scale = pow(2, _levelOfInterest); + int levelOffsetX = offsetX / scale; + int levelOffsetY = offsetY / scale; + + + + if (!downscaleByPowerOfTwo(resizedColor, resizedMask, input, inputMask, _levelOfInterest)) { + return false; + } - aliceVision::image::Image buf(current_color.Width(), current_color.Height()); - aliceVision::image::Image next_color(current_color.Width() / 2, current_color.Height() / 2); - aliceVision::image::Image next_mask(current_color.Width() / 2, current_color.Height() / 2); + BoundingBox bb; + bb.left = 0; + bb.top = 0; + bb.width = resizedColor.Width(); + bb.height = resizedColor.Height(); - convolveGaussian5x5(buf, current_color); - downscale(next_color, buf); + CachedImage destColor; + if (!destColor.createImage(_cacheManager, resizedColor.Width(), resizedColor.Height())) + { + return false; + } - for(int i = 0; i < next_mask.Height(); i++) - { - int di = i * 2; + if (!destColor.fill(image::RGBfColor(0.0f))) + { + return false; + } - for(int j = 0; j < next_mask.Width(); j++) - { - int dj = j * 2; + if (!destColor.assign(resizedColor, bb, bb)) + { + return false; + } - if(current_mask(di, dj) && current_mask(di, dj + 1) && current_mask(di + 1, dj) && - current_mask(di + 1, dj + 1)) - { - next_mask(i, j) = 255; - } - else - { - next_mask(i, j) = 0; - } - } - } - current_color = next_color; - current_mask = next_mask; - offset_x /= 2; - offset_y /= 2; + CachedImage destMask; + if (!destMask.createImage(_cacheManager, resizedMask.Width(), resizedMask.Height())) + { + return false; } + if (!destMask.fill(0)) + { + return false; + } - char filename[FILENAME_MAX]; - sprintf(filename, "/home/mmoc/test%d.exr", int(offset_x)); - image::writeImage(filename, current_color, image::EImageColorSpace::NO_CONVERSION); + if (!destMask.assign(resizedMask, bb, bb)) + { + return false; + } + - return _graphcut->append(current_color, current_mask, currentIndex, offset_x, offset_y); + return true; } bool HierarchicalGraphcutSeams::process() -{ - +{ /*if(!_graphcut->process()) { return false; diff --git a/src/aliceVision/panorama/seams.hpp b/src/aliceVision/panorama/seams.hpp index 7bd0e51933..d38d72be6e 100644 --- a/src/aliceVision/panorama/seams.hpp +++ b/src/aliceVision/panorama/seams.hpp @@ -5,6 +5,7 @@ #include "cachedImage.hpp" #include "graphcut.hpp" +#include "boundingBoxMap.hpp" namespace aliceVision { @@ -48,8 +49,9 @@ class WTASeams class HierarchicalGraphcutSeams { public: - HierarchicalGraphcutSeams(image::TileCacheManager::shared_ptr cacheManager, size_t outputWidth, size_t outputHeight, size_t levelOfInterest) + HierarchicalGraphcutSeams(image::TileCacheManager::shared_ptr cacheManager, const BoundingBoxMap & map, size_t outputWidth, size_t outputHeight, size_t levelOfInterest) : _cacheManager(cacheManager) + , _originalMap(map) , _outputWidth(outputWidth) , _outputHeight(outputHeight) , _levelOfInterest(levelOfInterest) @@ -86,7 +88,10 @@ class HierarchicalGraphcutSeams std::unique_ptr _graphcut; image::TileCacheManager::shared_ptr _cacheManager; CachedImage _labels; - + + //Original bounding box map at level 0 + BoundingBoxMap _originalMap; + size_t _levelOfInterest; size_t _outputWidth; size_t _outputHeight; diff --git a/src/software/pipeline/main_panoramaCompositing.cpp b/src/software/pipeline/main_panoramaCompositing.cpp index 3aee05a0f1..fc23b47dcd 100644 --- a/src/software/pipeline/main_panoramaCompositing.cpp +++ b/src/software/pipeline/main_panoramaCompositing.cpp @@ -32,6 +32,7 @@ #include #include #include +#include // These constants define the current software version. // They must be updated when the command line is changed. @@ -44,6 +45,42 @@ namespace po = boost::program_options; namespace bpt = boost::property_tree; namespace fs = boost::filesystem; +bool buildMap(BoundingBoxMap & map, const sfmData::SfMData& sfmData, const std::string & inputPath) +{ + for (const auto& viewIt : sfmData.getViews()) + { + if(!sfmData.isPoseAndIntrinsicDefined(viewIt.second.get())) + { + // skip unreconstructed views + continue; + } + + // Load mask + const std::string maskPath = (fs::path(inputPath) / (std::to_string(viewIt.first) + "_mask.exr")).string(); + oiio::ParamValueList metadata = image::readImageMetadata(maskPath); + + const std::size_t offsetX = metadata.find("AliceVision:offsetX")->get_int(); + const std::size_t offsetY = metadata.find("AliceVision:offsetY")->get_int(); + const std::size_t contentX = metadata.find("AliceVision:contentX")->get_int(); + const std::size_t contentY = metadata.find("AliceVision:contentY")->get_int(); + const std::size_t contentW = metadata.find("AliceVision:contentW")->get_int(); + const std::size_t contentH = metadata.find("AliceVision:contentH")->get_int(); + + BoundingBox bb; + bb.left = offsetX + contentX; + bb.top = offsetY + contentY; + bb.width = contentW; + bb.height = contentH; + + if (!map.append(viewIt.first, bb)) + { + return false; + } + } + + return true; +} + bool computeWTALabels(CachedImage & labels, image::TileCacheManager::shared_ptr& cacheManager, const sfmData::SfMData& sfmData, const std::string & inputPath, std::pair & panoramaSize) { WTASeams seams(panoramaSize.first, panoramaSize.second); @@ -86,7 +123,7 @@ bool computeWTALabels(CachedImage & labels, image::TileCacheManager::sha return true; } -bool computeGCLabels(CachedImage & labels, image::TileCacheManager::shared_ptr& cacheManager, const sfmData::SfMData& sfmData, const std::string & inputPath, std::pair & panoramaSize) +bool computeGCLabels(CachedImage & labels, image::TileCacheManager::shared_ptr& cacheManager, const sfmData::SfMData& sfmData, const std::string & inputPath, std::pair & panoramaSize, const BoundingBoxMap & map) { //Compute coarsest level possible for graph cut @@ -100,7 +137,7 @@ bool computeGCLabels(CachedImage & labels, image::TileCacheManager::shar for (int l = initial_level; l>= initial_level; l--) { - HierarchicalGraphcutSeams seams(cacheManager, panoramaSize.first, panoramaSize.second, l); + HierarchicalGraphcutSeams seams(cacheManager, map, panoramaSize.first, panoramaSize.second, l); if (!seams.initialize()) @@ -118,31 +155,31 @@ bool computeGCLabels(CachedImage & labels, image::TileCacheManager::shar seams.setMaximalDistance(100); } - /*for (const auto& viewIt : sfmData.getViews()) + for (const auto& viewIt : sfmData.getViews()) { if(!sfmData.isPoseAndIntrinsicDefined(viewIt.second.get())) { // skip unreconstructed views continue; } - + // Load mask const std::string maskPath = (fs::path(inputPath) / (std::to_string(viewIt.first) + "_mask.exr")).string(); ALICEVISION_LOG_INFO("Load mask with path " << maskPath); image::Image mask; image::readImage(maskPath, mask, image::EImageColorSpace::NO_CONVERSION); - // Get offset - oiio::ParamValueList metadata = image::readImageMetadata(maskPath); - const std::size_t offsetX = metadata.find("AliceVision:offsetX")->get_int(); - const std::size_t offsetY = metadata.find("AliceVision:offsetY")->get_int(); - // Load Color const std::string colorsPath = (fs::path(inputPath) / (std::to_string(viewIt.first) + ".exr")).string(); ALICEVISION_LOG_INFO("Load colors with path " << colorsPath); image::Image colors; image::readImage(colorsPath, colors, image::EImageColorSpace::NO_CONVERSION); + // Get offset + oiio::ParamValueList metadata = image::readImageMetadata(maskPath); + const std::size_t offsetX = metadata.find("AliceVision:offsetX")->get_int(); + const std::size_t offsetY = metadata.find("AliceVision:offsetY")->get_int(); + // Append to graph cut seams.append(colors, mask, viewIt.first, offsetX, offsetY); } @@ -152,7 +189,7 @@ bool computeGCLabels(CachedImage & labels, image::TileCacheManager::shar return false; } - labels = seams.getLabels();*/ + //labels = seams.getLabels(); } return true; @@ -292,6 +329,13 @@ int aliceVision_main(int argc, char** argv) return EXIT_FAILURE; } + BoundingBoxMap map; + if (!buildMap(map, sfmData, warpingFolder)) + { + ALICEVISION_LOG_ERROR("Error Building map"); + return EXIT_FAILURE; + } + CachedImage labels; if (!computeWTALabels(labels, cacheManager, sfmData, warpingFolder, panoramaSize)) { @@ -300,7 +344,7 @@ int aliceVision_main(int argc, char** argv) } - if (!computeGCLabels(labels, cacheManager, sfmData, warpingFolder, panoramaSize)) + if (!computeGCLabels(labels, cacheManager, sfmData, warpingFolder, panoramaSize, map)) { ALICEVISION_LOG_ERROR("Error computing graph cut labels"); return EXIT_FAILURE; diff --git a/src/software/pipeline/main_panoramaWarping.cpp b/src/software/pipeline/main_panoramaWarping.cpp index 66980b6494..22ef8397af 100644 --- a/src/software/pipeline/main_panoramaWarping.cpp +++ b/src/software/pipeline/main_panoramaWarping.cpp @@ -313,7 +313,7 @@ int aliceVision_main(int argc, char** argv) #pragma omp critical { - globalBbox.unionWith(map.getBoundingBox()); + globalBbox = globalBbox.unionWith(map.getBoundingBox()); } } From 6186a20ffa91b289a919e710db3f79c3671dd1da Mon Sep 17 00:00:00 2001 From: Fabien Servant Date: Thu, 15 Oct 2020 16:00:35 +0200 Subject: [PATCH 13/79] [panorama] first version of large graphcut --- src/aliceVision/image/io.cpp | 11 + src/aliceVision/image/io.hpp | 3 + src/aliceVision/panorama/CMakeLists.txt | 1 + src/aliceVision/panorama/boundingBox.cpp | 16 + src/aliceVision/panorama/boundingBox.hpp | 7 +- src/aliceVision/panorama/compositer.hpp | 123 +--- src/aliceVision/panorama/coordinatesMap.cpp | 20 +- src/aliceVision/panorama/graphcut.hpp | 613 +++++++++--------- src/aliceVision/panorama/imageOps.cpp | 9 +- src/aliceVision/panorama/imageOps.hpp | 122 ++++ src/aliceVision/panorama/seams.cpp | 78 ++- src/aliceVision/panorama/seams.hpp | 14 +- .../pipeline/main_panoramaCompositing.cpp | 23 +- .../pipeline/main_panoramaWarping.cpp | 34 +- 14 files changed, 592 insertions(+), 482 deletions(-) create mode 100644 src/aliceVision/panorama/boundingBox.cpp diff --git a/src/aliceVision/image/io.cpp b/src/aliceVision/image/io.cpp index 25dbe9eb8d..6270797a12 100644 --- a/src/aliceVision/image/io.cpp +++ b/src/aliceVision/image/io.cpp @@ -8,6 +8,7 @@ #include #include + #include #include #include @@ -506,6 +507,16 @@ void writeImage(const std::string& path, const Image& image, EIma writeImage(path, oiio::TypeDesc::UINT8, 1, image, imageColorSpace, metadata); } +void writeImage(const std::string& path, const Image& image, EImageColorSpace imageColorSpace, const oiio::ParamValueList& metadata) +{ + writeImage(path, oiio::TypeDesc::UINT32, 1, image, imageColorSpace, metadata); +} + +void writeImage(const std::string& path, const Image& image, EImageColorSpace imageColorSpace, const oiio::ParamValueList& metadata) +{ + writeImage(path, oiio::TypeDesc::UINT32, 1, image, imageColorSpace, metadata); +} + void writeImage(const std::string& path, const Image& image, EImageColorSpace imageColorSpace, const oiio::ParamValueList& metadata) { writeImage(path, oiio::TypeDesc::FLOAT, 4, image, imageColorSpace, metadata); diff --git a/src/aliceVision/image/io.hpp b/src/aliceVision/image/io.hpp index 87fc04f7f0..2a7916f2fc 100644 --- a/src/aliceVision/image/io.hpp +++ b/src/aliceVision/image/io.hpp @@ -9,6 +9,7 @@ #include #include +#include #include #include @@ -175,6 +176,8 @@ void readImage(const std::string& path, Image& image, EImageColorSpace */ void writeImage(const std::string& path, const Image& image, EImageColorSpace imageColorSpace, const oiio::ParamValueList& metadata = oiio::ParamValueList()); void writeImage(const std::string& path, const Image& image, EImageColorSpace imageColorSpace, const oiio::ParamValueList& metadata = oiio::ParamValueList()); +void writeImage(const std::string& path, const Image& image, EImageColorSpace imageColorSpace, const oiio::ParamValueList& metadata = oiio::ParamValueList()); +void writeImage(const std::string& path, const Image& image, EImageColorSpace imageColorSpace, const oiio::ParamValueList& metadata = oiio::ParamValueList()); void writeImage(const std::string& path, const Image& image, EImageColorSpace imageColorSpace, const oiio::ParamValueList& metadata = oiio::ParamValueList()); void writeImage(const std::string& path, const Image& image, EImageColorSpace imageColorSpace, const oiio::ParamValueList& metadata = oiio::ParamValueList()); void writeImage(const std::string& path, const Image& image, EImageColorSpace imageColorSpace, const oiio::ParamValueList& metadata = oiio::ParamValueList()); diff --git a/src/aliceVision/panorama/CMakeLists.txt b/src/aliceVision/panorama/CMakeLists.txt index 54a1fb80af..2d5798685d 100644 --- a/src/aliceVision/panorama/CMakeLists.txt +++ b/src/aliceVision/panorama/CMakeLists.txt @@ -22,6 +22,7 @@ set(panorama_files_headers set(panorama_files_sources warper.cpp gaussian.cpp + boundingBox.cpp coordinatesMap.cpp distance.cpp remapBbox.cpp diff --git a/src/aliceVision/panorama/boundingBox.cpp b/src/aliceVision/panorama/boundingBox.cpp new file mode 100644 index 0000000000..a624f6484b --- /dev/null +++ b/src/aliceVision/panorama/boundingBox.cpp @@ -0,0 +1,16 @@ +#include "boundingBox.hpp" + +namespace aliceVision +{ + +std::ostream& operator<<(std::ostream& os, const BoundingBox& in) +{ + os << in.left << " "; + os << in.top << " "; + os << in.width << " "; + os << in.height; + + return os; +} + +} \ No newline at end of file diff --git a/src/aliceVision/panorama/boundingBox.hpp b/src/aliceVision/panorama/boundingBox.hpp index 4d5b1e877d..e81b58ff39 100644 --- a/src/aliceVision/panorama/boundingBox.hpp +++ b/src/aliceVision/panorama/boundingBox.hpp @@ -3,6 +3,7 @@ #include #include #include +#include namespace aliceVision { @@ -84,8 +85,8 @@ struct BoundingBox int maxr = std::max(rt, ro); int maxb = std::max(bt, bo); - ret.width = maxr - left + 1; - ret.height = maxb - top + 1; + ret.width = maxr - ret.left + 1; + ret.height = maxb - ret.top + 1; return ret; } @@ -203,4 +204,6 @@ struct BoundingBox } }; +std::ostream& operator<<(std::ostream& os, const BoundingBox& in); + } \ No newline at end of file diff --git a/src/aliceVision/panorama/compositer.hpp b/src/aliceVision/panorama/compositer.hpp index 98c61b311c..3f2a2d1b12 100644 --- a/src/aliceVision/panorama/compositer.hpp +++ b/src/aliceVision/panorama/compositer.hpp @@ -3,132 +3,11 @@ #include #include "cachedImage.hpp" +#include "imageOps.hpp" namespace aliceVision { -template -bool loopyCachedImageAssign(CachedImage & output, const aliceVision::image::Image & input, const BoundingBox & assignedOutputBb, const BoundingBox & assignedInputBb) -{ - BoundingBox inputBb = assignedInputBb; - BoundingBox outputBb = assignedOutputBb; - - if (inputBb.width != outputBb.width) - { - return false; - } - - if (inputBb.height != outputBb.height) - { - return false; - } - - if (outputBb.getBottom() >= output.getHeight()) - { - outputBb.height = output.getHeight() - outputBb.top; - inputBb.height = outputBb.height; - } - - if (assignedOutputBb.getRight() < output.getWidth()) { - - if (!output.assign(input, inputBb, outputBb)) - { - return false; - } - } - else { - - int left_1 = assignedOutputBb.left; - int left_2 = 0; - int width1 = output.getWidth() - assignedOutputBb.left; - int width2 = input.Width() - width1; - - inputBb.left = 0; - outputBb.left = left_1; - inputBb.width = width1; - outputBb.width = width1; - - - if (!output.assign(input, inputBb, outputBb)) - { - return false; - } - - inputBb.left = width1; - outputBb.left = 0; - - //no overlap - int width2_clamped = std::min(width2, left_1); - inputBb.width = width2_clamped; - outputBb.width = width2_clamped; - if (width2_clamped == 0) return true; - - - if (!output.assign(input, inputBb, outputBb)) - { - return false; - } - } - - return true; -} - -template -bool loopyCachedImageExtract(aliceVision::image::Image & output, CachedImage & input, const BoundingBox & extractedInputBb) -{ - BoundingBox outputBb; - BoundingBox inputBb; - - outputBb.left = 0; - outputBb.top = 0; - outputBb.width = output.Width(); - outputBb.height = output.Height(); - - inputBb = extractedInputBb; - if (inputBb.getBottom() >= input.getHeight()) - { - inputBb.height = input.getHeight() - inputBb.top; - outputBb.height = inputBb.height; - } - - if (extractedInputBb.getRight() < input.getWidth()) - { - if (!input.extract(output, outputBb, inputBb)) - { - return false; - } - } - else - { - int left_1 = extractedInputBb.left; - int left_2 = 0; - int width_1 = input.getWidth() - extractedInputBb.left; - int width_2 = output.Width() - width_1; - - outputBb.left = 0; - inputBb.left = left_1; - outputBb.width = width_1; - inputBb.width = width_1; - - if (!input.extract(output, outputBb, inputBb)) - { - return false; - } - - outputBb.left = width_1; - inputBb.left = 0; - outputBb.width = width_2; - inputBb.width = width_2; - - if (!input.extract(output, outputBb, inputBb)) - { - return false; - } - } - - return true; -} - class Compositer { public: diff --git a/src/aliceVision/panorama/coordinatesMap.cpp b/src/aliceVision/panorama/coordinatesMap.cpp index acf2e86637..d9cd2af34b 100644 --- a/src/aliceVision/panorama/coordinatesMap.cpp +++ b/src/aliceVision/panorama/coordinatesMap.cpp @@ -13,20 +13,20 @@ bool CoordinatesMap::build(const std::pair& panoramaSize, const geomet _coordinates = aliceVision::image::Image(coarseBbox.width, coarseBbox.height, false); _mask = aliceVision::image::Image(coarseBbox.width, coarseBbox.height, true, 0); - size_t max_x = 0; - size_t max_y = 0; - size_t min_x = panoramaSize.first; - size_t min_y = panoramaSize.second; + int max_x = 0; + int max_y = 0; + int min_x = std::numeric_limits::max(); + int min_y = std::numeric_limits::max(); - for(size_t y = 0; y < coarseBbox.height; y++) + for(int y = 0; y < coarseBbox.height; y++) { - size_t cy = y + coarseBbox.top; + int cy = y + coarseBbox.top; - for(size_t x = 0; x < coarseBbox.width; x++) + for(int x = 0; x < coarseBbox.width; x++) { - size_t cx = x + coarseBbox.left; + int cx = x + coarseBbox.left; Vec3 ray = SphericalMapping::fromEquirectangular(Vec2(cx, cy), panoramaSize.first, panoramaSize.second); @@ -68,8 +68,8 @@ bool CoordinatesMap::build(const std::pair& panoramaSize, const geomet _boundingBox.left = min_x; _boundingBox.top = min_y; - _boundingBox.width = max_x - min_x + 1; - _boundingBox.height = max_y - min_y + 1; + _boundingBox.width = std::max(0, max_x - min_x + 1); + _boundingBox.height = std::max(0, max_y - min_y + 1); return true; } diff --git a/src/aliceVision/panorama/graphcut.hpp b/src/aliceVision/panorama/graphcut.hpp index 45d89537aa..4d28643e44 100644 --- a/src/aliceVision/panorama/graphcut.hpp +++ b/src/aliceVision/panorama/graphcut.hpp @@ -7,6 +7,7 @@ #include "distance.hpp" #include "boundingBox.hpp" +#include "imageOps.hpp" namespace aliceVision { @@ -219,9 +220,9 @@ class GraphcutSeams CachedImage color; CachedImage mask; }; + + using PixelInfo = std::map; - using PixelInfo = std::pair; - using ImageOwners = image::Image>; public: GraphcutSeams(size_t outputWidth, size_t outputHeight) @@ -274,153 +275,239 @@ class GraphcutSeams data.rect.left = offset_x; data.rect.top = offset_y; + _inputs[currentIndex] = data; - /* - BoundingBox rect; + return true; + } - rect.left = offset_x; - rect.top = offset_y; - rect.width = input.Width() + 1; - rect.height = input.Height() + 1; + void setMaximalDistance(int dist) + { + _maximal_distance_change = dist; + } - //Extend rect for borders - rect.dilate(3); - rect.clampLeft(); - rect.clampTop(); - rect.clampBottom(_owners.Height() - 1) + bool processInput(bool & hasChanged, InputData & input) + { + //Get bounding box of input in panoram + //Dilate to have some pixels outside of the input + BoundingBox localBbox = input.rect.dilate(3); + localBbox.clampLeft(); + localBbox.clampTop(); + localBbox.clampBottom(_labels.getHeight() - 1); + + //Output must keep a margin also + BoundingBox outputBbox = input.rect; + outputBbox.left = input.rect.left - localBbox.left; + outputBbox.top = input.rect.top - localBbox.top; - _rects[currentIndex] = rect; - - //_owners will get for each pixel of the panorama a list of pixels - //in the sources which may have seen this point. - - for(int i = 0; i < input.Height(); i++) + image::Image localLabels(localBbox.width, localBbox.height); + if (!loopyCachedImageExtract(localLabels, _labels, localBbox)) { + return false; + } - int di = i + offset_y; + image::Image graphCutInput(localBbox.width, localBbox.height, true); + + for (auto & otherInput : _inputs) + { + BoundingBox otherBbox = otherInput.second.rect; + BoundingBox otherBboxLoop = otherInput.second.rect; + otherBboxLoop.left = otherBbox.left - _outputWidth; + BoundingBox otherBboxLoopRight = otherInput.second.rect; + otherBboxLoopRight.left = otherBbox.left + _outputWidth; + + BoundingBox otherInputBbox = otherBbox; + otherInputBbox.left = 0; + otherInputBbox.top = 0; + + BoundingBox intersection = localBbox.intersectionWith(otherBbox); + BoundingBox intersectionLoop = localBbox.intersectionWith(otherBboxLoop); + BoundingBox intersectionLoopRight = localBbox.intersectionWith(otherBboxLoopRight); + if (intersection.isEmpty() && intersectionLoop.isEmpty() && intersectionLoopRight.isEmpty()) + { + continue; + } + + image::Image otherColor(otherInputBbox.width, otherInputBbox.height); + if (!otherInput.second.color.extract(otherColor, otherInputBbox, otherInputBbox)) + { + return false; + } - for(int j = 0; j < input.Width(); j++) + image::Image otherMask(otherInputBbox.width, otherInputBbox.height); + if (!otherInput.second.mask.extract(otherMask, otherInputBbox, otherInputBbox)) { + return false; + } - if(!inputMask(i, j)) - continue; - int dj = j + offset_x; - if(dj >= _owners.Width()) - { - dj = dj - _owners.Width(); - } + if (!intersection.isEmpty()) + { + BoundingBox interestOther = intersection; + interestOther.left -= otherBbox.left; + interestOther.top -= otherBbox.top; - PixelInfo info; - info.first = currentIndex; - info.second = input(i, j); + BoundingBox interestThis = intersection; + interestThis.left -= localBbox.left; + interestThis.top -= localBbox.top; - // If too far away from seam, do not add a contender - int dist = _distancesSeams(di, dj); - if (dist > _maximal_distance_change + 10) + for (int y = 0; y < intersection.height; y++) { - continue; - } + int y_other = interestOther.top + y; + int y_current = interestThis.top + y; - _owners(di, dj).push_back(info); + for (int x = 0; x < intersection.width; x++) + { + int x_other = interestOther.left + x; + int x_current = interestThis.left + x; + + if (!otherMask(y_other, x_other)) + { + continue; + } + + PixelInfo & pix = graphCutInput(y_current, x_current); + pix[otherInput.first] = otherColor(y_other, x_other); + } + } } - }*/ - return true; - } + if (!intersectionLoop.isEmpty()) + { + BoundingBox interestOther = intersectionLoop; + interestOther.left -= otherBboxLoop.left; + interestOther.top -= otherBboxLoop.top; - void setMaximalDistance(int dist) - { - _maximal_distance_change = dist; - } + BoundingBox interestThis = intersectionLoop; + interestThis.left -= localBbox.left; + interestThis.top -= localBbox.top; - bool process() - { + + for (int y = 0; y < intersectionLoop.height; y++) + { + int y_other = interestOther.top + y; + int y_current = interestThis.top + y; - /*for(int i = 0; i < 10; i++) - { + for (int x = 0; x < intersectionLoop.width; x++) + { + int x_other = interestOther.left + x; + int x_current = interestThis.left + x; - // For each possible label, try to extends its domination on the label's world - bool change = false; + if (!otherMask(y_other, x_other)) + { + continue; + } - for(auto & info : _rects) - { + PixelInfo & pix = graphCutInput(y_current, x_current); + pix[otherInput.first] = otherColor(y_other, x_other); + } + } + } - ALICEVISION_LOG_INFO("Graphcut expansion (iteration " << i << ") for label " << info.first); + if (!intersectionLoopRight.isEmpty()) + { + BoundingBox interestOther = intersectionLoopRight; + interestOther.left -= otherBboxLoopRight.left; + interestOther.top -= otherBboxLoopRight.top; - int p1 = info.second.left; - int w1 = info.second.width; - int p2 = 0; - int w2 = 0; + BoundingBox interestThis = intersectionLoopRight; + interestThis.left -= localBbox.left; + interestThis.top -= localBbox.top; - if(p1 + w1 > _labels.Width()) + + for (int y = 0; y < intersectionLoopRight.height; y++) { - w1 = _labels.Width() - p1; - p2 = 0; - w2 = info.second.width - w1; - } + int y_other = interestOther.top + y; + int y_current = interestThis.top + y; - Eigen::Matrix backup_1 = _labels.block(info.second.top, p1, info.second.height, w1); - Eigen::Matrix backup_2 = _labels.block(info.second.top, p2, info.second.height, w2); + for (int x = 0; x < intersectionLoopRight.width; x++) + { + int x_other = interestOther.left + x; + int x_current = interestThis.left + x; - double base_cost = cost(info.first); - alphaExpansion(info.first); - double new_cost = cost(info.first); + if (!otherMask(y_other, x_other)) + { + continue; + } - if(new_cost > base_cost) - { - _labels.block(info.second.top, p1, info.second.height, w1) = backup_1; - _labels.block(info.second.top, p2, info.second.height, w2) = backup_2; - } - else if(new_cost < base_cost) - { - change = true; + PixelInfo & pix = graphCutInput(y_current, x_current); + pix[otherInput.first] = otherColor(y_other, x_other); + } } } + } - if(!change) + double costBefore = cost(localLabels, graphCutInput, input.id); + if (!alphaExpansion(localLabels, graphCutInput, input.id)) + { + return false; + } + double costAfter = cost(localLabels, graphCutInput, input.id); + + hasChanged = false; + if (costAfter < costBefore) + { + hasChanged = true; + BoundingBox inputBb = localBbox; + inputBb.left = 0; + inputBb.top = 0; + + if (!loopyCachedImageAssign(_labels, localLabels, localBbox, inputBb)) { - break; + return false; } - }*/ + } + return true; } - double cost(IndexT currentLabel) + bool process() { + for (int i = 0; i < 3; i++) + { + std::cout << "**************************" << std::endl; + // For each possible label, try to extends its domination on the label's world + bool change = false; - //BoundingBox rect = _rects[currentLabel]; + for (auto & info : _inputs) + { + bool lchange = true; + if (!processInput(lchange, info.second)) + { + return false; + } - double cost = 0.0; + change &= lchange; + } - /*for(int i = 0; i < rect.height - 1; i++) - { + if (!change) + { + break; + } + } - int y = rect.top + i; - int yp = y + 1; + _labels.writeImage("/home/mmoc/test.exr"); - for(int j = 0; j < rect.width; j++) - { + return true; + } - int x = rect.left + j; - if(x >= _owners.Width()) - { - x = x - _owners.Width(); - } + double cost(const image::Image localLabels, const image::Image & input, IndexT currentLabel) + { + double cost = 0.0; + for (int y = 0; y < input.Height() - 1; y++) + { + for(int x = 0; x < input.Width() - 1; x++) + { int xp = x + 1; - if(xp >= _owners.Width()) - { - xp = xp - _owners.Width(); - } + int yp = y + 1; - IndexT label = _labels(y, x); - IndexT labelx = _labels(y, xp); - IndexT labely = _labels(yp, x); + IndexT label = localLabels(y, x); + IndexT labelx = localLabels(y, xp); + IndexT labely = localLabels(yp, x); if(label == UndefinedIndexT) continue; @@ -429,77 +516,69 @@ class GraphcutSeams if(labely == UndefinedIndexT) continue; - if(label == labelx) - { - continue; - } - image::RGBfColor CColorLC; image::RGBfColor CColorLX; image::RGBfColor CColorLY; + image::RGBfColor XColorLC; + image::RGBfColor XColorLX; + image::RGBfColor YColorLC; + image::RGBfColor YColorLY; + bool hasYLC = false; + bool hasYLY = false; + bool hasXLC = false; + bool hasXLX = false; bool hasCLC = false; bool hasCLX = false; bool hasCLY = false; - for(int l = 0; l < _owners(y, x).size(); l++) - { - if(_owners(y, x)[l].first == label) - { - hasCLC = true; - CColorLC = _owners(y, x)[l].second; - } - if(_owners(y, x)[l].first == labelx) - { - hasCLX = true; - CColorLX = _owners(y, x)[l].second; - } - - if(_owners(y, x)[l].first == labely) - { - hasCLY = true; - CColorLY = _owners(y, x)[l].second; - } + auto it = input(y, x).find(label); + if (it != input(y, x).end()) + { + hasCLC = true; + CColorLC = it->second; } - image::RGBfColor XColorLC; - image::RGBfColor XColorLX; - bool hasXLC = false; - bool hasXLX = false; + it = input(y, x).find(labelx); + if (it != input(y, x).end()) + { + hasCLX = true; + CColorLX = it->second; + } - for(int l = 0; l < _owners(y, xp).size(); l++) + it = input(y, x).find(labely); + if (it != input(y, x).end()) { - if(_owners(y, xp)[l].first == label) - { - hasXLC = true; - XColorLC = _owners(y, xp)[l].second; - } + hasCLY = true; + CColorLY = it->second; + } - if(_owners(y, xp)[l].first == labelx) - { - hasXLX = true; - XColorLX = _owners(y, xp)[l].second; - } + it = input(y, xp).find(label); + if (it != input(y, xp).end()) + { + hasXLC = true; + XColorLC = it->second; } - image::RGBfColor YColorLC; - image::RGBfColor YColorLY; - bool hasYLC = false; - bool hasYLY = false; + it = input(y, xp).find(labelx); + if (it != input(y, xp).end()) + { + hasXLX = true; + XColorLX = it->second; + } - for(int l = 0; l < _owners(yp, x).size(); l++) + it = input(yp, x).find(label); + if (it != input(yp, x).end()) { - if(_owners(yp, x)[l].first == label) - { - hasYLC = true; - YColorLC = _owners(yp, x)[l].second; - } + hasYLC = true; + YColorLC = it->second; + } - if(_owners(yp, x)[l].first == labely) - { - hasYLY = true; - YColorLY = _owners(yp, x)[l].second; - } + it = input(yp, x).find(labely); + if (it != input(yp, x).end()) + { + hasYLY = true; + YColorLY = it->second; } if(!hasCLC || !hasXLX || !hasYLY) @@ -532,52 +611,41 @@ class GraphcutSeams cost += (XColorLC - XColorLX).norm(); cost += (YColorLC - YColorLY).norm(); } - }*/ + } return cost; } - bool alphaExpansion(IndexT currentLabel) + bool alphaExpansion(image::Image & labels, const image::Image & input, IndexT currentLabel) { - /* - BoundingBox rect = _rects[currentLabel]; - - image::Image mask(rect.width, rect.height, true, 0); - image::Image ids(rect.width, rect.height, true, -1); - image::Image color_label(rect.width, rect.height, true, image::RGBfColor(0.0f, 0.0f, 0.0f)); - image::Image color_other(rect.width, rect.height, true, image::RGBfColor(0.0f, 0.0f, 0.0f)); + image::Image mask(labels.Width(), labels.Height(), true, 0); + image::Image ids(labels.Width(), labels.Height(), true, -1); + image::Image color_label(labels.Width(), labels.Height(), true, image::RGBfColor(0.0f, 0.0f, 0.0f)); + image::Image color_other(labels.Width(), labels.Height(), true, image::RGBfColor(0.0f, 0.0f, 0.0f)); // Compute distance map to seams - image::Image distanceMap(rect.width, rect.height); + image::Image distanceMap(labels.Width(), labels.Height()); { - image::Image binarizedWorld(rect.width, rect.height); + image::Image binarizedWorld(labels.Width(), labels.Height()); - for(int i = 0; i < rect.height; i++) + for(int y = 0; y < labels.Height(); y++) { - int y = rect.top + i; - - for(int j = 0; j < rect.width; j++) + for(int x = 0; x < labels.Width(); x++) { + IndexT label = labels(y, x); - int x = rect.left + j; - if(x >= _owners.Width()) - { - x = x - _owners.Width(); - } - - IndexT label = _original_labels(y, x); if(label == currentLabel) { - binarizedWorld(i, j) = 1; + binarizedWorld(y, x) = 1; } else { - binarizedWorld(i, j) = 0; + binarizedWorld(y, x) = 0; } } } - image::Image seams(rect.width, rect.height); + image::Image seams(labels.Width(), labels.Height()); if(!computeSeamsMap(seams, binarizedWorld)) { return false; @@ -589,139 +657,108 @@ class GraphcutSeams } } - - //A warped input has valid pixels only in some parts of the final image. - //Rect is the bounding box of these valid pixels. - //Let's build a mask : - // - 0 if the pixel is not viewed by anyone - // - 1 if the pixel is viewed by the current label alpha - // - 2 if the pixel is viewed by *another* label and this label is marked as current valid label - // - 3 if the pixel is 1 + 2 : the pixel is not selected as alpha territory, but alpha is looking at it - - for(int i = 0; i < rect.height; i++) + for (int y = 0; y < labels.Height(); y++) { - - int y = rect.top + i; - - for(int j = 0; j < rect.width; j++) + for (int x = 0; x < labels.Width(); x++) { + IndexT label = labels(y, x); - int x = rect.left + j; - if(x >= _owners.Width()) - { - x = x - _owners.Width(); - } - - std::vector& infos = _owners(y, x); - IndexT label = _labels(y, x); + int dist = distanceMap(y, x); image::RGBfColor currentColor; image::RGBfColor otherColor; - int dist = distanceMap(i, j); - - // Loop over observations - for(int l = 0; l < infos.size(); l++) + auto it = input(y, x).find(currentLabel); + if (it != input(y, x).end()) { + currentColor = it->second; + mask(y, x) = 1; + } - if(dist > _maximal_distance_change) + if (label != currentLabel) + { + it = input(y, x).find(label); + if (it != input(y, x).end()) { + otherColor = it->second; - if(infos[l].first == label) - { - if(label == currentLabel) - { - mask(i, j) = 1; - currentColor = infos[l].second; - } - else - { - mask(i, j) = 2; - otherColor = infos[l].second; - } - } - } - else - { - if(infos[l].first == currentLabel) + if (dist > _maximal_distance_change) { - mask(i, j) |= 1; - currentColor = infos[l].second; + mask(y, x) = 2; } - else if(infos[l].first == label) + else { - mask(i, j) |= 2; - otherColor = infos[l].second; + mask(y, x) |= 2; } } } // If the pixel may be a new kingdom for alpha - if(mask(i, j) == 1) + if(mask(y, x) == 1) { - color_label(i, j) = currentColor; - color_other(i, j) = currentColor; + color_label(y, x) = currentColor; + color_other(y, x) = currentColor; } - else if(mask(i, j) == 2) + else if(mask(y, x) == 2) { - color_label(i, j) = otherColor; - color_other(i, j) = otherColor; + color_label(y, x) = otherColor; + color_other(y, x) = otherColor; } - else if(mask(i, j) == 3) + else if(mask(y, x) == 3) { - color_label(i, j) = currentColor; - color_other(i, j) = otherColor; + color_label(y, x) = currentColor; + color_other(y, x) = otherColor; } } - } + } // The rectangle is a grid. // However we want to ignore a lot of pixel. // Let's create an index per valid pixels for graph cut reference int count = 0; - for(int i = 0; i < rect.height; i++) + for(int y = 0; y < labels.Height(); y++) { - for(int j = 0; j < rect.width; j++) + for(int x = 0; x < labels.Width(); x++) { - if(mask(i, j) == 0) + if(mask(y, x) == 0) { continue; } - ids(i, j) = count; + ids(y, x) = count; count++; } - } + } //Create graph MaxFlow_AdjList gc(count); size_t countValid = 0; - for(int i = 0; i < rect.height; i++) + for(int y = 0; y < labels.Height(); y++) { - for(int j = 0; j < rect.width; j++) + for(int x = 0; x < labels.Width(); x++) { // If this pixel is not valid, ignore - if(mask(i, j) == 0) + if(mask(y, x) == 0) { continue; } // Get this pixel ID - int node_id = ids(i, j); + int node_id = ids(y, x); - int im1 = std::max(i - 1, 0); - int jm1 = std::max(j - 1, 0); - int ip1 = std::min(i + 1, rect.height - 1); - int jp1 = std::min(j + 1, rect.width - 1); + int ym1 = std::max(y - 1, 0); + int xm1 = std::max(x - 1, 0); + int yp1 = std::min(y + 1, labels.Height() - 1); + int xp1 = std::min(x + 1, labels.Width() - 1); - if(mask(i, j) == 1) + if(mask(y, x) == 1) { - // Only add nodes close to borders - if(mask(im1, jm1) == 1 && mask(im1, j) == 1 && mask(im1, jp1) == 1 && mask(i, jm1) == 1 && - mask(i, jp1) == 1 && mask(ip1, jm1) == 1 && mask(ip1, j) == 1 && mask(ip1, jp1) == 1) + if(mask(ym1, xm1) == 1 && mask(ym1, x) == 1 && mask(ym1, xp1) == 1 && + mask(y, xm1) == 1 && mask(y, xp1) == 1 && + mask(yp1, xm1) == 1 && mask(yp1, x) == 1 && mask(yp1, xp1) == 1) { continue; } @@ -732,11 +769,12 @@ class GraphcutSeams //from alpha territoy is infinitly costly (impossible). gc.addNodeToSource(node_id, 100000); } - else if(mask(i, j) == 2) + else if(mask(y, x) == 2) { // Only add nodes close to borders - if(mask(im1, jm1) == 2 && mask(im1, j) == 2 && mask(im1, jp1) == 2 && mask(i, jm1) == 2 && - mask(i, jp1) == 2 && mask(ip1, jm1) == 2 && mask(ip1, j) == 2 && mask(ip1, jp1) == 2) + if(mask(ym1, xm1) == 2 && mask(ym1, x) == 2 && mask(ym1, xp1) == 2 && + mask(y, xm1) == 2 && mask(y, xp1) == 2 && + mask(yp1, xm1) == 2 && mask(yp1, x) == 2 && mask(yp1, xp1) == 2) { continue; } @@ -746,7 +784,7 @@ class GraphcutSeams //from ennemy territory is infinitly costly (impossible). gc.addNodeToSink(node_id, 100000); } - else if(mask(i, j) == 3) + else if(mask(y, x) == 3) { // This pixel is seen by both alpha and enemies but is owned by ennemy. @@ -760,6 +798,7 @@ class GraphcutSeams } } + if(countValid == 0) { // We have no possibility for territory expansion @@ -767,38 +806,38 @@ class GraphcutSeams return true; } + // Loop over alpha bounding box. // Let's define the transition cost. // When two neighboor pixels have different labels, there is a seam (border) cost. // Graph cut will try to make sure the territory will have a minimal border cost - - for(int i = 0; i < rect.height; i++) + + for(int y = 0; y < labels.Height(); y++) { - for(int j = 0; j < rect.width; j++) + for(int x = 0; x < labels.Width(); x++) { - if(mask(i, j) == 0) + if(mask(y, x) == 0) { continue; } - int node_id = ids(i, j); + int node_id = ids(y, x); // Make sure it is possible to estimate this horizontal border - if(i < mask.Height() - 1) + if(y < mask.Height() - 1) { - // Make sure the other pixel is owned by someone - if(mask(i + 1, j)) + if(mask(y + 1, x)) { - int other_node_id = ids(i + 1, j); + int other_node_id = ids(y + 1, x); float w = 1000; - if(((mask(i, j) & 1) && (mask(i + 1, j) & 2)) || ((mask(i, j) & 2) && (mask(i + 1, j) & 1))) + if(((mask(y, x) & 1) && (mask(y + 1, x) & 2)) || ((mask(y, x) & 2) && (mask(y + 1, x) & 1))) { - float d1 = (color_label(i, j) - color_other(i, j)).norm(); - float d2 = (color_label(i + 1, j) - color_other(i + 1, j)).norm(); + float d1 = (color_label(y, x) - color_other(y, x)).norm(); + float d2 = (color_label(y + 1, x) - color_other(y + 1, x)).norm(); d1 = std::min(2.0f, d1); d2 = std::min(2.0f, d2); @@ -810,19 +849,19 @@ class GraphcutSeams } } - if(j < mask.Width() - 1) + if(x < mask.Width() - 1) { - if(mask(i, j + 1)) + if(mask(y, x + 1)) { - int other_node_id = ids(i, j + 1); + int other_node_id = ids(y, x + 1); float w = 1000; - if(((mask(i, j) & 1) && (mask(i, j + 1) & 2)) || ((mask(i, j) & 2) && (mask(i, j + 1) & 1))) + if(((mask(y, x) & 1) && (mask(y, x + 1) & 2)) || ((mask(y, x) & 2) && (mask(y, x + 1) & 1))) { - float d1 = (color_label(i, j) - color_other(i, j)).norm(); - float d2 = (color_label(i, j + 1) - color_other(i, j + 1)).norm(); + float d1 = (color_label(y, x) - color_other(y, x)).norm(); + float d2 = (color_label(y, x + 1) - color_other(y, x + 1)).norm(); w = (d1 + d2) * 100.0 + 1.0; } @@ -835,22 +874,12 @@ class GraphcutSeams gc.compute(); int changeCount = 0; - for(int i = 0; i < rect.height; i++) + for(int y = 0; y < labels.Height(); y++) { - - int y = rect.top + i; - - for(int j = 0; j < rect.width; j++) + for(int x = 0; x < labels.Width(); x++) { - - int x = rect.left + j; - if(x >= _owners.Width()) - { - x = x - _owners.Width(); - } - - IndexT label = _labels(y, x); - int id = ids(i, j); + IndexT label = labels(y, x); + int id = ids(y, x); if(gc.isSource(id)) { @@ -860,10 +889,10 @@ class GraphcutSeams changeCount++; } - _labels(y, x) = currentLabel; + labels(y, x) = currentLabel; } } - }*/ + } return true; } @@ -880,9 +909,7 @@ class GraphcutSeams int _outputWidth; int _outputHeight; size_t _maximal_distance_change; - CachedImage _labels; - ImageOwners _owners; }; } // namespace aliceVision \ No newline at end of file diff --git a/src/aliceVision/panorama/imageOps.cpp b/src/aliceVision/panorama/imageOps.cpp index 56efd84480..2795d61806 100644 --- a/src/aliceVision/panorama/imageOps.cpp +++ b/src/aliceVision/panorama/imageOps.cpp @@ -1,5 +1,6 @@ #include "imageOps.hpp" #include "gaussian.hpp" +#include "feathering.hpp" namespace aliceVision { @@ -38,7 +39,13 @@ void removeNegativeValues(CachedImage & img) bool downscaleByPowerOfTwo(image::Image & output, image::Image & outputMask, const image::Image & input, const image::Image & inputMask, const int timesDividedBy2) { - image::Image currentColor = input; + image::Image currentColor(input.Width(), input.Height()); + + if (!feathering(currentColor, input, inputMask)) + { + return false; + } + image::Image currentMask = inputMask; for(int l = 1; l <= timesDividedBy2; l++) diff --git a/src/aliceVision/panorama/imageOps.hpp b/src/aliceVision/panorama/imageOps.hpp index 28e2320bee..e08ad80e24 100644 --- a/src/aliceVision/panorama/imageOps.hpp +++ b/src/aliceVision/panorama/imageOps.hpp @@ -116,4 +116,126 @@ bool downscaleByPowerOfTwo(image::Image & output, image::Image void removeNegativeValues(CachedImage& img); +template +bool loopyCachedImageAssign(CachedImage & output, const aliceVision::image::Image & input, const BoundingBox & assignedOutputBb, const BoundingBox & assignedInputBb) +{ + BoundingBox inputBb = assignedInputBb; + BoundingBox outputBb = assignedOutputBb; + + if (inputBb.width != outputBb.width) + { + return false; + } + + if (inputBb.height != outputBb.height) + { + return false; + } + + if (outputBb.getBottom() >= output.getHeight()) + { + outputBb.height = output.getHeight() - outputBb.top; + inputBb.height = outputBb.height; + } + + if (assignedOutputBb.getRight() < output.getWidth()) { + + if (!output.assign(input, inputBb, outputBb)) + { + return false; + } + } + else { + + int left_1 = assignedOutputBb.left; + int left_2 = 0; + int width1 = output.getWidth() - assignedOutputBb.left; + int width2 = input.Width() - width1; + + inputBb.left = 0; + outputBb.left = left_1; + inputBb.width = width1; + outputBb.width = width1; + + + if (!output.assign(input, inputBb, outputBb)) + { + return false; + } + + inputBb.left = width1; + outputBb.left = 0; + + //no overlap + int width2_clamped = std::min(width2, left_1); + inputBb.width = width2_clamped; + outputBb.width = width2_clamped; + if (width2_clamped == 0) return true; + + + if (!output.assign(input, inputBb, outputBb)) + { + return false; + } + } + + return true; +} + +template +bool loopyCachedImageExtract(aliceVision::image::Image & output, CachedImage & input, const BoundingBox & extractedInputBb) +{ + BoundingBox outputBb; + BoundingBox inputBb; + + outputBb.left = 0; + outputBb.top = 0; + outputBb.width = output.Width(); + outputBb.height = output.Height(); + + inputBb = extractedInputBb; + if (inputBb.getBottom() >= input.getHeight()) + { + inputBb.height = input.getHeight() - inputBb.top; + outputBb.height = inputBb.height; + } + + if (extractedInputBb.getRight() < input.getWidth()) + { + if (!input.extract(output, outputBb, inputBb)) + { + return false; + } + } + else + { + int left_1 = extractedInputBb.left; + int left_2 = 0; + int width_1 = input.getWidth() - extractedInputBb.left; + int width_2 = output.Width() - width_1; + + outputBb.left = 0; + inputBb.left = left_1; + outputBb.width = width_1; + inputBb.width = width_1; + + if (!input.extract(output, outputBb, inputBb)) + { + return false; + } + + outputBb.left = width_1; + inputBb.left = 0; + outputBb.width = width_2; + inputBb.width = width_2; + + if (!input.extract(output, outputBb, inputBb)) + { + return false; + } + } + + return true; +} + } // namespace aliceVision \ No newline at end of file diff --git a/src/aliceVision/panorama/seams.cpp b/src/aliceVision/panorama/seams.cpp index 1d3b2e90d5..065f798b21 100644 --- a/src/aliceVision/panorama/seams.cpp +++ b/src/aliceVision/panorama/seams.cpp @@ -307,8 +307,8 @@ bool HierarchicalGraphcutSeams::setOriginalLabels(CachedImage& labels) } bool HierarchicalGraphcutSeams::append(const aliceVision::image::Image& input, - const aliceVision::image::Image& inputMask, IndexT currentIndex, - size_t offsetX, size_t offsetY) + const aliceVision::image::Image& inputMask, + IndexT currentIndex, size_t offsetX, size_t offsetY) { image::Image resizedColor; image::Image resizedMask; @@ -317,8 +317,6 @@ bool HierarchicalGraphcutSeams::append(const aliceVision::image::Imageappend(destColor, destMask, currentIndex, levelOffsetX, levelOffsetY); } bool HierarchicalGraphcutSeams::process() { - /*if(!_graphcut->process()) + if(!_graphcut->process()) { return false; } - image::Image current_labels = _graphcut->getLabels(); + CachedImage smallLabels = _graphcut->getLabels(); - for(int l = _levelOfInterest - 1; l >= 0; l--) - { + int scale = pow(2, _levelOfInterest); - int nw = current_labels.Width() * 2; - int nh = current_labels.Height() * 2; - if(l == 0) - { - nw = _outputWidth; - nh = _outputHeight; - } + int processingSize = 256; + int largeSize = 256 * scale; - aliceVision::image::Image next_label(nw, nh); - for(int i = 0; i < nh; i++) + for (int i = 0; i < smallLabels.getHeight(); i+= processingSize) + { + for (int j = 0; j < smallLabels.getWidth(); j+= processingSize) { - int hi = i / 2; + BoundingBox smallBb; + smallBb.left = j; + smallBb.top = i; + smallBb.width = processingSize; + smallBb.height = processingSize; + smallBb.clampRight(smallLabels.getWidth() - 1); + smallBb.clampBottom(smallLabels.getHeight() - 1); + + BoundingBox smallInputBb; + smallInputBb.left = 0; + smallInputBb.top = 0; + smallInputBb.width = smallBb.width; + smallInputBb.height = smallBb.height; + - for(int j = 0; j < nw; j++) + image::Image smallView(smallBb.width, smallBb.height); + if (!smallLabels.extract(smallView, smallInputBb, smallBb)) { - int hj = j / 2; + return false; + } - next_label(i, j) = current_labels(hi, hj); + BoundingBox largeBb; + largeBb.left = smallBb.left * scale; + largeBb.top = smallBb.top * scale; + largeBb.width = smallBb.width * scale; + largeBb.height = smallBb.height * scale; + + BoundingBox largeInputBb; + largeInputBb.left = 0; + largeInputBb.top = 0; + largeInputBb.width = largeBb.width; + largeInputBb.height = largeBb.height; + + image::Image largeView(largeBb.width, largeBb.height); + for (int y = 0; y < largeBb.height; y++) + { + for (int x = 0; x < largeBb.width; x++) + { + largeView(y, x) = smallView(y / scale, x / scale); + } } - } - current_labels = next_label; + if (!_labels.assign(largeView, largeInputBb, largeBb)) + { + return false; + } + } } - _labels = current_labels;*/ return true; } diff --git a/src/aliceVision/panorama/seams.hpp b/src/aliceVision/panorama/seams.hpp index d38d72be6e..30a5088577 100644 --- a/src/aliceVision/panorama/seams.hpp +++ b/src/aliceVision/panorama/seams.hpp @@ -30,8 +30,8 @@ class WTASeams bool initialize(image::TileCacheManager::shared_ptr & cacheManager); bool append(const aliceVision::image::Image& inputMask, - const aliceVision::image::Image& inputWeights, IndexT currentIndex, size_t offset_x, - size_t offset_y); + const aliceVision::image::Image& inputWeights, + IndexT currentIndex, size_t offset_x, size_t offset_y); CachedImage & getLabels() { @@ -49,9 +49,8 @@ class WTASeams class HierarchicalGraphcutSeams { public: - HierarchicalGraphcutSeams(image::TileCacheManager::shared_ptr cacheManager, const BoundingBoxMap & map, size_t outputWidth, size_t outputHeight, size_t levelOfInterest) + HierarchicalGraphcutSeams(image::TileCacheManager::shared_ptr cacheManager, size_t outputWidth, size_t outputHeight, size_t levelOfInterest) : _cacheManager(cacheManager) - , _originalMap(map) , _outputWidth(outputWidth) , _outputHeight(outputHeight) , _levelOfInterest(levelOfInterest) @@ -74,8 +73,8 @@ class HierarchicalGraphcutSeams } virtual bool append(const aliceVision::image::Image& input, - const aliceVision::image::Image& inputMask, IndexT currentIndex, size_t offset_x, - size_t offset_y); + const aliceVision::image::Image& inputMask, + IndexT currentIndex, size_t offset_x, size_t offset_y); bool process(); @@ -89,9 +88,6 @@ class HierarchicalGraphcutSeams image::TileCacheManager::shared_ptr _cacheManager; CachedImage _labels; - //Original bounding box map at level 0 - BoundingBoxMap _originalMap; - size_t _levelOfInterest; size_t _outputWidth; size_t _outputHeight; diff --git a/src/software/pipeline/main_panoramaCompositing.cpp b/src/software/pipeline/main_panoramaCompositing.cpp index fc23b47dcd..994721267c 100644 --- a/src/software/pipeline/main_panoramaCompositing.cpp +++ b/src/software/pipeline/main_panoramaCompositing.cpp @@ -120,10 +120,12 @@ bool computeWTALabels(CachedImage & labels, image::TileCacheManager::sha labels = seams.getLabels(); + + return true; } -bool computeGCLabels(CachedImage & labels, image::TileCacheManager::shared_ptr& cacheManager, const sfmData::SfMData& sfmData, const std::string & inputPath, std::pair & panoramaSize, const BoundingBoxMap & map) +bool computeGCLabels(CachedImage & labels, image::TileCacheManager::shared_ptr& cacheManager, const sfmData::SfMData& sfmData, const std::string & inputPath, std::pair & panoramaSize) { //Compute coarsest level possible for graph cut @@ -134,10 +136,10 @@ bool computeGCLabels(CachedImage & labels, image::TileCacheManager::shar initial_level = int(ceil(log2(ratio))); } - for (int l = initial_level; l>= initial_level; l--) + for (int l = initial_level; l>= 0; l--) { - HierarchicalGraphcutSeams seams(cacheManager, map, panoramaSize.first, panoramaSize.second, l); + HierarchicalGraphcutSeams seams(cacheManager, panoramaSize.first, panoramaSize.second, l); if (!seams.initialize()) @@ -180,8 +182,12 @@ bool computeGCLabels(CachedImage & labels, image::TileCacheManager::shar const std::size_t offsetX = metadata.find("AliceVision:offsetX")->get_int(); const std::size_t offsetY = metadata.find("AliceVision:offsetY")->get_int(); + // Append to graph cut - seams.append(colors, mask, viewIt.first, offsetX, offsetY); + if (!seams.append(colors, mask, viewIt.first, offsetX, offsetY)) + { + return false; + } } if (!seams.process()) @@ -189,7 +195,7 @@ bool computeGCLabels(CachedImage & labels, image::TileCacheManager::shar return false; } - //labels = seams.getLabels(); + labels = seams.getLabels(); } return true; @@ -344,7 +350,7 @@ int aliceVision_main(int argc, char** argv) } - if (!computeGCLabels(labels, cacheManager, sfmData, warpingFolder, panoramaSize, map)) + if (!computeGCLabels(labels, cacheManager, sfmData, warpingFolder, panoramaSize)) { ALICEVISION_LOG_ERROR("Error computing graph cut labels"); return EXIT_FAILURE; @@ -352,8 +358,6 @@ int aliceVision_main(int argc, char** argv) labels.writeImage("/home/mmoc/labels.exr"); - return EXIT_SUCCESS; - //Get a list of views ordered by their image scale std::vector> viewOrderedByScale; @@ -381,6 +385,8 @@ int aliceVision_main(int argc, char** argv) size_t pos = 0; + std::cout << "compositing" << std::endl; + for(const auto & view : viewOrderedByScale) { IndexT viewId = view->getViewId(); @@ -432,6 +438,7 @@ int aliceVision_main(int argc, char** argv) return EXIT_FAILURE; } + std::cout << pos << std::endl; compositer.append(source, mask, seams, offsetX, offsetY, imageContent); } diff --git a/src/software/pipeline/main_panoramaWarping.cpp b/src/software/pipeline/main_panoramaWarping.cpp index 22ef8397af..890b67733d 100644 --- a/src/software/pipeline/main_panoramaWarping.cpp +++ b/src/software/pipeline/main_panoramaWarping.cpp @@ -215,6 +215,7 @@ int aliceVision_main(int argc, char** argv) rangeStart = 0; rangeSize = int(viewsOrderedByName.size()); } + ALICEVISION_LOG_DEBUG("Range to compute: rangeStart=" << rangeStart << ", rangeSize=" << rangeSize); @@ -260,6 +261,11 @@ int aliceVision_main(int argc, char** argv) continue; } + /*if (view.getViewId() != 828178699) + { + continue; + }*/ + ALICEVISION_LOG_INFO("[" << int(i) + 1 - rangeStart << "/" << rangeSize << "] Processing view " << view.getViewId() << " (" << i + 1 << "/" << viewsOrderedByName.size() << ")"); // Get intrinsics and extrinsics @@ -296,14 +302,9 @@ int aliceVision_main(int argc, char** argv) } #pragma omp parallel for - for (int i = 0; i < boxes.size(); i++) { + for (int boxId = 0; boxId < boxes.size(); boxId++) { - BoundingBox localBbox = boxes[i]; - - /*int stillToProcess = coarseBbox.width - localBbox.left; - if (stillToProcess < tileSize) { - localBbox.width = stillToProcess; - }*/ + BoundingBox localBbox = boxes[boxId]; // Prepare coordinates map CoordinatesMap map; @@ -311,12 +312,23 @@ int aliceVision_main(int argc, char** argv) continue; } + if (map.getBoundingBox().isEmpty()) + { + continue; + } + + #pragma omp critical { globalBbox = globalBbox.unionWith(map.getBoundingBox()); } } + //Rare case ... When all boxes valid are after the loop + if (globalBbox.left >= panoramaSize.first) + { + globalBbox.left -= panoramaSize.first; + } // Update bounding box BoundingBox snappedGlobalBbox; @@ -403,17 +415,15 @@ int aliceVision_main(int argc, char** argv) } - //#pragma omp parallel for + #pragma omp parallel for { - for (int i = 0; i < boxes.size(); i++) + for (int boxId = 0; boxId < boxes.size(); boxId++) { - BoundingBox localBbox = boxes[i]; + BoundingBox localBbox = boxes[boxId]; int x = localBbox.left - snappedGlobalBbox.left; int y = localBbox.top - snappedGlobalBbox.top; - - // Prepare coordinates map CoordinatesMap map; if (!map.build(panoramaSize, camPose, *(intrinsic.get()), localBbox)) From bf4e1c7b4d23d25e1ca2dd7c039884bac54f43fe Mon Sep 17 00:00:00 2001 From: Fabien Servant Date: Fri, 16 Oct 2020 11:13:12 +0200 Subject: [PATCH 14/79] [panorama] some small changes --- src/aliceVision/image/cache.cpp | 28 +++++++++++++++++++ .../pipeline/main_panoramaWarping.cpp | 19 ++++++------- 2 files changed, 37 insertions(+), 10 deletions(-) diff --git a/src/aliceVision/image/cache.cpp b/src/aliceVision/image/cache.cpp index e1383fa63b..87d66b2cb4 100644 --- a/src/aliceVision/image/cache.cpp +++ b/src/aliceVision/image/cache.cpp @@ -308,8 +308,36 @@ _tileWidth(tileWidth), _tileHeight(tileHeight) { } +static unsigned int bitCount (unsigned int value) +{ + unsigned int count = 0; + + while (value > 0) + { + if ((value & 1) == 1) + { + count++; + } + + value >>= 1; + } + + return count; +} + + std::shared_ptr TileCacheManager::create(const std::string & path_storage, size_t tileWidth, size_t tileHeight, size_t maxTilesPerIndex) { + if (bitCount(tileWidth) != 1) + { + return nullptr; + } + + if (bitCount(tileHeight) != 1) + { + return nullptr; + } + TileCacheManager * obj = new TileCacheManager(path_storage, tileWidth, tileHeight, maxTilesPerIndex); return std::shared_ptr(obj); diff --git a/src/software/pipeline/main_panoramaWarping.cpp b/src/software/pipeline/main_panoramaWarping.cpp index 890b67733d..b9ca83fccf 100644 --- a/src/software/pipeline/main_panoramaWarping.cpp +++ b/src/software/pipeline/main_panoramaWarping.cpp @@ -237,8 +237,8 @@ int aliceVision_main(int argc, char** argv) } // Make sure panorama size has required size properties - double max_scale = 1.0 / pow(2.0, 10); - panoramaSize.first = int(ceil(double(panoramaSize.first) * max_scale) / max_scale); + /*double max_scale = 1.0 / pow(2.0, 10); + panoramaSize.first = int(ceil(double(panoramaSize.first) * max_scale) / max_scale);*/ panoramaSize.second = panoramaSize.first / 2; ALICEVISION_LOG_INFO("Choosen panorama size : " << panoramaSize.first << "x" << panoramaSize.second); @@ -261,11 +261,6 @@ int aliceVision_main(int argc, char** argv) continue; } - /*if (view.getViewId() != 828178699) - { - continue; - }*/ - ALICEVISION_LOG_INFO("[" << int(i) + 1 - rangeStart << "/" << rangeSize << "] Processing view " << view.getViewId() << " (" << i + 1 << "/" << viewsOrderedByName.size() << ")"); // Get intrinsics and extrinsics @@ -295,8 +290,12 @@ int aliceVision_main(int argc, char** argv) BoundingBox localBbox; localBbox.left = x + snappedCoarseBbox.left; localBbox.top = y + snappedCoarseBbox.top; - localBbox.width = tileSize; + localBbox.width = tileSize; localBbox.height = tileSize; + + localBbox.clampRight(snappedCoarseBbox.getRight()); + localBbox.clampBottom(snappedCoarseBbox.getBottom()); + boxes.push_back(localBbox); } } @@ -335,7 +334,7 @@ int aliceVision_main(int argc, char** argv) // Once again, snap to grid snappedGlobalBbox = globalBbox; - snappedGlobalBbox.snapToGrid(tileSize); + //snappedGlobalBbox.snapToGrid(tileSize); if (snappedGlobalBbox.width <= 0 || snappedGlobalBbox.height <= 0) continue; @@ -414,7 +413,7 @@ int aliceVision_main(int argc, char** argv) } } - + #pragma omp parallel for { for (int boxId = 0; boxId < boxes.size(); boxId++) From c2e27dd97e85b53a1ea24d9233ab72d6d7d8637c Mon Sep 17 00:00:00 2001 From: Fabien Servant Date: Fri, 16 Oct 2020 11:32:45 +0200 Subject: [PATCH 15/79] [panorama] adding new bugs correction --- src/aliceVision/panorama/cachedImage.cpp | 10 +++++----- src/aliceVision/panorama/graphcut.hpp | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/aliceVision/panorama/cachedImage.cpp b/src/aliceVision/panorama/cachedImage.cpp index b146d9512a..2baea431be 100644 --- a/src/aliceVision/panorama/cachedImage.cpp +++ b/src/aliceVision/panorama/cachedImage.cpp @@ -13,7 +13,7 @@ bool CachedImage::writeImage(const std::string& path) return false; } - oiio::ImageSpec spec(_memoryWidth, _memoryHeight, 4, oiio::TypeDesc::FLOAT); + oiio::ImageSpec spec(_width, _height, 4, oiio::TypeDesc::FLOAT); spec.tile_width = _tileSize; spec.tile_height = _tileSize; @@ -56,7 +56,7 @@ bool CachedImage::writeImage(const std::string& path) return false; } - oiio::ImageSpec spec(_memoryWidth, _memoryHeight, 3, oiio::TypeDesc::FLOAT); + oiio::ImageSpec spec(_width, _height, 3, oiio::TypeDesc::FLOAT); spec.tile_width = _tileSize; spec.tile_height = _tileSize; @@ -99,7 +99,7 @@ bool CachedImage::writeImage(const std::string& path) return false; } - oiio::ImageSpec spec(_memoryWidth, _memoryHeight, 1, oiio::TypeDesc::UINT32); + oiio::ImageSpec spec(_width, _height, 1, oiio::TypeDesc::UINT32); spec.tile_width = _tileSize; spec.tile_height = _tileSize; @@ -142,7 +142,7 @@ bool CachedImage::writeImage(const std::string& path) return false; } - oiio::ImageSpec spec(_memoryWidth, _memoryHeight, 1, oiio::TypeDesc::FLOAT); + oiio::ImageSpec spec(_width, _height, 1, oiio::TypeDesc::FLOAT); spec.tile_width = _tileSize; spec.tile_height = _tileSize; @@ -185,7 +185,7 @@ bool CachedImage::writeImage(const std::string& path) return false; } - oiio::ImageSpec spec(_memoryWidth, _memoryHeight, 1, oiio::TypeDesc::UINT8); + oiio::ImageSpec spec(_width, _height, 1, oiio::TypeDesc::UINT8); spec.tile_width = _tileSize; spec.tile_height = _tileSize; diff --git a/src/aliceVision/panorama/graphcut.hpp b/src/aliceVision/panorama/graphcut.hpp index 4d28643e44..0000a63be0 100644 --- a/src/aliceVision/panorama/graphcut.hpp +++ b/src/aliceVision/panorama/graphcut.hpp @@ -479,8 +479,8 @@ class GraphcutSeams { return false; } - - change &= lchange; + + change |= lchange; } if (!change) From 490865fdb908e139efc33ad9adc169858717b897 Mon Sep 17 00:00:00 2001 From: Fabien Servant Date: Fri, 16 Oct 2020 14:53:50 +0200 Subject: [PATCH 16/79] [panorama] graphcut improvement --- src/aliceVision/panorama/graphcut.hpp | 104 +++++++++++------- .../pipeline/main_panoramaCompositing.cpp | 7 +- 2 files changed, 69 insertions(+), 42 deletions(-) diff --git a/src/aliceVision/panorama/graphcut.hpp b/src/aliceVision/panorama/graphcut.hpp index 0000a63be0..4272d64a28 100644 --- a/src/aliceVision/panorama/graphcut.hpp +++ b/src/aliceVision/panorama/graphcut.hpp @@ -210,6 +210,7 @@ bool computeSeamsMap(image::Image& seams, const image::Image localLabels(localBbox.width, localBbox.height); if (!loopyCachedImageExtract(localLabels, _labels, localBbox)) { return false; } + // Compute distance map to borders of the input seams + image::Image distanceMap(localLabels.Width(), localLabels.Height()); + { + image::Image binarizedWorld(localLabels.Width(), localLabels.Height()); + + for(int y = 0; y < localLabels.Height(); y++) + { + for(int x = 0; x < localLabels.Width(); x++) + { + IndexT label = localLabels(y, x); + + if(label == input.id) + { + binarizedWorld(y, x) = 1; + } + else + { + binarizedWorld(y, x) = 0; + } + } + } + + image::Image seams(localLabels.Width(), localLabels.Height()); + if(!computeSeamsMap(seams, binarizedWorld)) + { + return false; + } + + if(!computeDistanceMap(distanceMap, seams)) + { + return false; + } + } + + + //Build the input image::Image graphCutInput(localBbox.width, localBbox.height, true); for (auto & otherInput : _inputs) @@ -367,6 +404,12 @@ class GraphcutSeams { continue; } + + float dist = sqrt(float(distanceMap(y_current, x_current))); + if (dist > _maximal_distance_change) + { + continue; + } PixelInfo & pix = graphCutInput(y_current, x_current); pix[otherInput.first] = otherColor(y_other, x_other); @@ -400,6 +443,12 @@ class GraphcutSeams continue; } + float dist = sqrt(float(distanceMap(y_current, x_current))); + if (dist > _maximal_distance_change) + { + continue; + } + PixelInfo & pix = graphCutInput(y_current, x_current); pix[otherInput.first] = otherColor(y_other, x_other); } @@ -432,6 +481,12 @@ class GraphcutSeams continue; } + float dist = sqrt(float(distanceMap(y_current, x_current))); + if (dist > _maximal_distance_change) + { + continue; + } + PixelInfo & pix = graphCutInput(y_current, x_current); pix[otherInput.first] = otherColor(y_other, x_other); } @@ -440,12 +495,13 @@ class GraphcutSeams } double costBefore = cost(localLabels, graphCutInput, input.id); - if (!alphaExpansion(localLabels, graphCutInput, input.id)) + if (!alphaExpansion(localLabels, distanceMap, graphCutInput, input.id)) { return false; } double costAfter = cost(localLabels, graphCutInput, input.id); + hasChanged = false; if (costAfter < costBefore) { @@ -466,12 +522,13 @@ class GraphcutSeams bool process() { - for (int i = 0; i < 3; i++) + for (int i = 0; i < 10; i++) { std::cout << "**************************" << std::endl; // For each possible label, try to extends its domination on the label's world bool change = false; + int pos = 0; for (auto & info : _inputs) { bool lchange = true; @@ -489,7 +546,9 @@ class GraphcutSeams } } - _labels.writeImage("/home/mmoc/test.exr"); + char filename[512]; + sprintf(filename, "/home/mmoc/test%d.exr", _outputWidth); + _labels.writeImage(filename); return true; } @@ -616,46 +675,13 @@ class GraphcutSeams return cost; } - bool alphaExpansion(image::Image & labels, const image::Image & input, IndexT currentLabel) + bool alphaExpansion(image::Image & labels, const image::Image & distanceMap, const image::Image & input, IndexT currentLabel) { image::Image mask(labels.Width(), labels.Height(), true, 0); image::Image ids(labels.Width(), labels.Height(), true, -1); image::Image color_label(labels.Width(), labels.Height(), true, image::RGBfColor(0.0f, 0.0f, 0.0f)); image::Image color_other(labels.Width(), labels.Height(), true, image::RGBfColor(0.0f, 0.0f, 0.0f)); - // Compute distance map to seams - image::Image distanceMap(labels.Width(), labels.Height()); - { - image::Image binarizedWorld(labels.Width(), labels.Height()); - - for(int y = 0; y < labels.Height(); y++) - { - for(int x = 0; x < labels.Width(); x++) - { - IndexT label = labels(y, x); - - if(label == currentLabel) - { - binarizedWorld(y, x) = 1; - } - else - { - binarizedWorld(y, x) = 0; - } - } - } - - image::Image seams(labels.Width(), labels.Height()); - if(!computeSeamsMap(seams, binarizedWorld)) - { - return false; - } - - if(!computeDistanceMap(distanceMap, seams)) - { - return false; - } - } for (int y = 0; y < labels.Height(); y++) { @@ -663,7 +689,7 @@ class GraphcutSeams { IndexT label = labels(y, x); - int dist = distanceMap(y, x); + float dist = sqrt(float(distanceMap(y, x))); image::RGBfColor currentColor; image::RGBfColor otherColor; diff --git a/src/software/pipeline/main_panoramaCompositing.cpp b/src/software/pipeline/main_panoramaCompositing.cpp index 994721267c..4bd20a52b4 100644 --- a/src/software/pipeline/main_panoramaCompositing.cpp +++ b/src/software/pipeline/main_panoramaCompositing.cpp @@ -349,15 +349,16 @@ int aliceVision_main(int argc, char** argv) return EXIT_FAILURE; } + labels.writeImage("/home/mmoc/labels_wta.exr"); - if (!computeGCLabels(labels, cacheManager, sfmData, warpingFolder, panoramaSize)) + /* if (!computeGCLabels(labels, cacheManager, sfmData, warpingFolder, panoramaSize)) { ALICEVISION_LOG_ERROR("Error computing graph cut labels"); return EXIT_FAILURE; } - labels.writeImage("/home/mmoc/labels.exr"); - + labels.writeImage("/home/mmoc/labels_gc.exr"); +*/ //Get a list of views ordered by their image scale std::vector> viewOrderedByScale; From b9f66cb17b19f65d39fbf52d5c96b1df0560382c Mon Sep 17 00:00:00 2001 From: Fabien Servant Date: Sat, 17 Oct 2020 10:15:10 +0200 Subject: [PATCH 17/79] [panorama] correct loop --- src/aliceVision/panorama/imageOps.hpp | 58 +++++++++---------- src/aliceVision/panorama/laplacianPyramid.cpp | 41 ++++++++----- .../pipeline/main_panoramaCompositing.cpp | 15 ++++- 3 files changed, 69 insertions(+), 45 deletions(-) diff --git a/src/aliceVision/panorama/imageOps.hpp b/src/aliceVision/panorama/imageOps.hpp index e08ad80e24..a60e794bc7 100644 --- a/src/aliceVision/panorama/imageOps.hpp +++ b/src/aliceVision/panorama/imageOps.hpp @@ -138,15 +138,16 @@ bool loopyCachedImageAssign(CachedImage & output, const aliceVision::image::I inputBb.height = outputBb.height; } - if (assignedOutputBb.getRight() < output.getWidth()) { + if (assignedOutputBb.getRight() < output.getWidth()) + { if (!output.assign(input, inputBb, outputBb)) { return false; } } - else { - + else + { int left_1 = assignedOutputBb.left; int left_2 = 0; int width1 = output.getWidth() - assignedOutputBb.left; @@ -192,46 +193,45 @@ bool loopyCachedImageExtract(aliceVision::image::Image & output, CachedImage< outputBb.top = 0; outputBb.width = output.Width(); outputBb.height = output.Height(); - + inputBb = extractedInputBb; if (inputBb.getBottom() >= input.getHeight()) { inputBb.height = input.getHeight() - inputBb.top; outputBb.height = inputBb.height; } - + if (extractedInputBb.getRight() < input.getWidth()) { - if (!input.extract(output, outputBb, inputBb)) + if (!input.extract(output, outputBb, extractedInputBb)) { return false; } } else - { - int left_1 = extractedInputBb.left; - int left_2 = 0; - int width_1 = input.getWidth() - extractedInputBb.left; - int width_2 = output.Width() - width_1; - - outputBb.left = 0; - inputBb.left = left_1; - outputBb.width = width_1; - inputBb.width = width_1; - - if (!input.extract(output, outputBb, inputBb)) - { - return false; - } - - outputBb.left = width_1; - inputBb.left = 0; - outputBb.width = width_2; - inputBb.width = width_2; - - if (!input.extract(output, outputBb, inputBb)) + { + int availableWidth = output.Width(); + while (availableWidth > 0) { - return false; + inputBb.clampRight(input.getWidth() - 1); + int extractedWidth = std::min(inputBb.width, availableWidth); + + + inputBb.width = extractedWidth; + outputBb.width = extractedWidth; + + if (!input.extract(output, outputBb, inputBb)) + { + return false; + } + + //Update the bouding box for output + outputBb.left += extractedWidth; + availableWidth -= extractedWidth; + + //All the input is available. + inputBb.left = 0; + inputBb.width = input.getWidth(); } } diff --git a/src/aliceVision/panorama/laplacianPyramid.cpp b/src/aliceVision/panorama/laplacianPyramid.cpp index b8d21b7ab3..7e35d76f32 100644 --- a/src/aliceVision/panorama/laplacianPyramid.cpp +++ b/src/aliceVision/panorama/laplacianPyramid.cpp @@ -556,28 +556,43 @@ bool LaplacianPyramid::rebuild(CachedImage& output) { for (int x = 0; x < _levels[halfLevel].getWidth(); x += processingSize) { + //Estimate the Bounding box + //Make sure we are not processing outside of the image bounding box BoundingBox extractedBb; extractedBb.left = x; extractedBb.top = y; extractedBb.width = processingSize; extractedBb.height = processingSize; - extractedBb.clampLeft(); - extractedBb.clampTop(); extractedBb.clampRight(_levels[halfLevel].getWidth() - 1); extractedBb.clampBottom(_levels[halfLevel].getHeight() - 1); - + + //Add borders to this bounding box for filtering BoundingBox dilatedBb = extractedBb.dilate(borderSize); - dilatedBb.clampLeft(); dilatedBb.clampTop(); - dilatedBb.clampBottom(_levels[halfLevel].getHeight() - 1); - + dilatedBb.clampBottom(_levels[halfLevel].getHeight() - 1); + + //If the box has a negative left border, + //it is equivalent (with the loop, to shifting at the end) + int leftBorder = extractedBb.left - dilatedBb.left; + if (dilatedBb.left < 0) + { + dilatedBb.left = _levels[halfLevel].getWidth() + dilatedBb.left; + } + BoundingBox doubleDilatedBb = dilatedBb.doubleSize(); BoundingBox doubleBb = extractedBb.doubleSize(); + + std::cout << extractedBb << std::endl; + std::cout << dilatedBb << std::endl; + std::cout << doubleDilatedBb << std::endl; + std::cout << doubleBb << std::endl; + + aliceVision::image::Image extracted(dilatedBb.width, dilatedBb.height); if (!loopyCachedImageExtract(extracted, _levels[halfLevel], dilatedBb)) { - return false; + return false; } aliceVision::image::Image extractedNext(doubleDilatedBb.width, doubleDilatedBb.height); @@ -602,13 +617,13 @@ bool LaplacianPyramid::rebuild(CachedImage& output) addition(extractedNext, extractedNext, buf2); - BoundingBox inputBb; - inputBb.left = doubleBb.left - doubleDilatedBb.left; - inputBb.top = doubleBb.top - doubleDilatedBb.top; - inputBb.width = doubleBb.width; - inputBb.height = doubleBb.height; + BoundingBox inputAssigmentBb = doubleBb; + inputAssigmentBb.left = 2 * leftBorder; + inputAssigmentBb.top = 0; + + std::cout << inputAssigmentBb << std::endl; - if (!loopyCachedImageAssign(_levels[currentLevel], extractedNext, doubleBb, inputBb)) + if (!loopyCachedImageAssign(_levels[currentLevel], extractedNext, doubleBb, inputAssigmentBb)) { return false; } diff --git a/src/software/pipeline/main_panoramaCompositing.cpp b/src/software/pipeline/main_panoramaCompositing.cpp index 4bd20a52b4..f3e95ce985 100644 --- a/src/software/pipeline/main_panoramaCompositing.cpp +++ b/src/software/pipeline/main_panoramaCompositing.cpp @@ -115,7 +115,10 @@ bool computeWTALabels(CachedImage & labels, image::TileCacheManager::sha image::Image weights; image::readImage(weightsPath, weights, image::EImageColorSpace::NO_CONVERSION); - seams.append(mask, weights, viewIt.first, offsetX, offsetY); + if (!seams.append(mask, weights, viewIt.first, offsetX, offsetY)) + { + return false; + } } labels = seams.getLabels(); @@ -440,10 +443,16 @@ int aliceVision_main(int argc, char** argv) } std::cout << pos << std::endl; - compositer.append(source, mask, seams, offsetX, offsetY, imageContent); + if (!compositer.append(source, mask, seams, offsetX, offsetY, imageContent)) + { + return EXIT_FAILURE; + } } - compositer.terminate(); + if (!compositer.terminate()) + { + return EXIT_FAILURE; + } compositer.save(outputPanorama); From 2e95d511ace80cba1b343321ceff30b294c0ab54 Mon Sep 17 00:00:00 2001 From: Fabien Servant Date: Mon, 19 Oct 2020 14:22:23 +0200 Subject: [PATCH 18/79] [panorama] graphcut correction --- src/aliceVision/panorama/graphcut.hpp | 87 ++++- src/aliceVision/panorama/imageOps.cpp | 53 --- src/aliceVision/panorama/imageOps.hpp | 81 +++- .../panorama/laplacianCompositer.hpp | 60 --- src/aliceVision/panorama/laplacianPyramid.cpp | 43 +- src/aliceVision/panorama/seams.cpp | 368 ++++++++++-------- src/aliceVision/panorama/seams.hpp | 20 +- 7 files changed, 367 insertions(+), 345 deletions(-) diff --git a/src/aliceVision/panorama/graphcut.hpp b/src/aliceVision/panorama/graphcut.hpp index 4272d64a28..c454b519e5 100644 --- a/src/aliceVision/panorama/graphcut.hpp +++ b/src/aliceVision/panorama/graphcut.hpp @@ -288,7 +288,7 @@ class GraphcutSeams _maximal_distance_change = dist; } - bool processInput(bool & hasChanged, InputData & input) + bool processInput(double & newCost, InputData & input) { //Get bounding box of input in panoram //Dilate to have some pixels outside of the input @@ -494,18 +494,45 @@ class GraphcutSeams } } - double costBefore = cost(localLabels, graphCutInput, input.id); + //Because of upscaling, some labels may be incorrect + //Some pixels may be affected to labels they don't see. + for (int y = 0; y < graphCutInput.Height(); y++) + { + for (int x = 0; x < graphCutInput.Width(); x++) + { + IndexT label = localLabels(y, x); + + if (label == UndefinedIndexT) + { + continue; + } + + PixelInfo & pix = graphCutInput(y, x); + + if (pix.size() == 0) + { + localLabels(y, x) = UndefinedIndexT; + continue; + } + + auto it = pix.find(label); + if (it == pix.end()) + { + localLabels(y, x) = pix.begin()->first; + } + } + } + + double oldCost = cost(localLabels, graphCutInput, input.id); if (!alphaExpansion(localLabels, distanceMap, graphCutInput, input.id)) { return false; } - double costAfter = cost(localLabels, graphCutInput, input.id); + newCost = cost(localLabels, graphCutInput, input.id); - hasChanged = false; - if (costAfter < costBefore) + if (newCost < oldCost) { - hasChanged = true; BoundingBox inputBb = localBbox; inputBb.left = 0; inputBb.top = 0; @@ -515,6 +542,10 @@ class GraphcutSeams return false; } } + else + { + newCost = oldCost; + } return true; @@ -522,25 +553,38 @@ class GraphcutSeams bool process() { + std::map costs; + for (auto & info : _inputs) + { + costs[info.first] = std::numeric_limits::max(); + } + for (int i = 0; i < 10; i++) { std::cout << "**************************" << std::endl; // For each possible label, try to extends its domination on the label's world - bool change = false; + + bool hasChange = false; int pos = 0; for (auto & info : _inputs) { - bool lchange = true; - if (!processInput(lchange, info.second)) + double cost; + if (!processInput(cost, info.second)) { return false; } - - change |= lchange; + + if (costs[info.first] != cost) + { + std::cout << costs[info.first] << " ----> " << cost << std::endl; + + costs[info.first] = cost; + hasChange = true; + } } - if (!change) + if (!hasChange) { break; } @@ -665,10 +709,17 @@ class GraphcutSeams YColorLC = YColorLY; } - cost += (CColorLC - CColorLX).norm(); - cost += (CColorLC - CColorLY).norm(); - cost += (XColorLC - XColorLX).norm(); - cost += (YColorLC - YColorLY).norm(); + double c1 = (CColorLC - CColorLX).norm(); + double c2 = (CColorLC - CColorLY).norm(); + double c3 = (XColorLC - XColorLX).norm(); + double c4 = (YColorLC - YColorLY).norm(); + + c1 = std::min(2.0, c1); + c2 = std::min(2.0, c2); + c3 = std::min(2.0, c3); + c4 = std::min(2.0, c4); + + cost += c1 + c2 + c3 + c4; } } @@ -888,6 +939,10 @@ class GraphcutSeams { float d1 = (color_label(y, x) - color_other(y, x)).norm(); float d2 = (color_label(y, x + 1) - color_other(y, x + 1)).norm(); + + d1 = std::min(2.0f, d1); + d2 = std::min(2.0f, d2); + w = (d1 + d2) * 100.0 + 1.0; } diff --git a/src/aliceVision/panorama/imageOps.cpp b/src/aliceVision/panorama/imageOps.cpp index 2795d61806..f37102621c 100644 --- a/src/aliceVision/panorama/imageOps.cpp +++ b/src/aliceVision/panorama/imageOps.cpp @@ -37,57 +37,4 @@ void removeNegativeValues(CachedImage & img) ); } -bool downscaleByPowerOfTwo(image::Image & output, image::Image & outputMask, const image::Image & input, const image::Image & inputMask, const int timesDividedBy2) -{ - image::Image currentColor(input.Width(), input.Height()); - - if (!feathering(currentColor, input, inputMask)) - { - return false; - } - - image::Image currentMask = inputMask; - - for(int l = 1; l <= timesDividedBy2; l++) - { - - aliceVision::image::Image buf(currentColor.Width(), currentColor.Height()); - aliceVision::image::Image nextColor(currentColor.Width() / 2, currentColor.Height() / 2); - aliceVision::image::Image nextMask(currentColor.Width() / 2, currentColor.Height() / 2); - - //Convolve + divide - convolveGaussian5x5(buf, currentColor); - downscale(nextColor, buf); - - //Just nearest neighboor divide for mask - for(int i = 0; i < nextMask.Height(); i++) - { - int di = i * 2; - - for(int j = 0; j < nextMask.Width(); j++) - { - int dj = j * 2; - - if(currentMask(di, dj) && currentMask(di, dj + 1) - && currentMask(di + 1, dj) && currentMask(di + 1, dj + 1)) - { - nextMask(i, j) = 255; - } - else - { - nextMask(i, j) = 0; - } - } - } - - currentColor = nextColor; - currentMask = nextMask; - } - - output = currentColor; - outputMask = currentMask; - - return true; -} - } // namespace aliceVision \ No newline at end of file diff --git a/src/aliceVision/panorama/imageOps.hpp b/src/aliceVision/panorama/imageOps.hpp index a60e794bc7..bb0dee4e7f 100644 --- a/src/aliceVision/panorama/imageOps.hpp +++ b/src/aliceVision/panorama/imageOps.hpp @@ -112,8 +112,6 @@ bool addition(aliceVision::image::Image& AplusB, const aliceVision::image::Im return true; } -bool downscaleByPowerOfTwo(image::Image & output, image::Image & outputMask, const image::Image & input, const image::Image & inputMask, const int timesDividedBy2); - void removeNegativeValues(CachedImage& img); template @@ -132,12 +130,6 @@ bool loopyCachedImageAssign(CachedImage & output, const aliceVision::image::I return false; } - if (outputBb.getBottom() >= output.getHeight()) - { - outputBb.height = output.getHeight() - outputBb.top; - inputBb.height = outputBb.height; - } - if (assignedOutputBb.getRight() < output.getWidth()) { @@ -192,18 +184,13 @@ bool loopyCachedImageExtract(aliceVision::image::Image & output, CachedImage< outputBb.left = 0; outputBb.top = 0; outputBb.width = output.Width(); - outputBb.height = output.Height(); + outputBb.height = std::min(extractedInputBb.height, output.Height()); inputBb = extractedInputBb; - if (inputBb.getBottom() >= input.getHeight()) - { - inputBb.height = input.getHeight() - inputBb.top; - outputBb.height = inputBb.height; - } - if (extractedInputBb.getRight() < input.getWidth()) + if (inputBb.getRight() < input.getWidth()) { - if (!input.extract(output, outputBb, extractedInputBb)) + if (!input.extract(output, outputBb, inputBb)) { return false; } @@ -238,4 +225,66 @@ bool loopyCachedImageExtract(aliceVision::image::Image & output, CachedImage< return true; } + +template +bool makeImagePyramidCompatible(image::Image& output, + size_t& out_offset_x, size_t& out_offset_y, + const image::Image& input, + size_t offset_x, size_t offset_y, + size_t num_levels) +{ + + if(num_levels == 0) + { + return false; + } + + double max_scale = 1.0 / pow(2.0, num_levels - 1); + + double low_offset_x = double(offset_x) * max_scale; + double low_offset_y = double(offset_y) * max_scale; + + /*Make sure offset is integer even at the lowest level*/ + double corrected_low_offset_x = floor(low_offset_x); + double corrected_low_offset_y = floor(low_offset_y); + + /*Add some borders on the top and left to make sure mask can be smoothed*/ + corrected_low_offset_x = std::max(0.0, corrected_low_offset_x - 3.0); + corrected_low_offset_y = std::max(0.0, corrected_low_offset_y - 3.0); + + /*Compute offset at largest level*/ + out_offset_x = size_t(corrected_low_offset_x / max_scale); + out_offset_y = size_t(corrected_low_offset_y / max_scale); + + /*Compute difference*/ + double doffset_x = double(offset_x) - double(out_offset_x); + double doffset_y = double(offset_y) - double(out_offset_y); + + /* update size with border update */ + double large_width = double(input.Width()) + doffset_x; + double large_height = double(input.Height()) + doffset_y; + + /* compute size at largest scale */ + double low_width = large_width * max_scale; + double low_height = large_height * max_scale; + + /*Make sure width is integer event at the lowest level*/ + double corrected_low_width = ceil(low_width); + double corrected_low_height = ceil(low_height); + + /*Add some borders on the right and bottom to make sure mask can be smoothed*/ + corrected_low_width = corrected_low_width + 3; + corrected_low_height = corrected_low_height + 3; + + /*Compute size at largest level*/ + size_t width = size_t(corrected_low_width / max_scale); + size_t height = size_t(corrected_low_height / max_scale); + + output = image::Image(width, height, true, T(0.0f)); + output.block(doffset_y, doffset_x, input.Height(), input.Width()) = input; + + return true; +} + + } // namespace aliceVision \ No newline at end of file diff --git a/src/aliceVision/panorama/laplacianCompositer.hpp b/src/aliceVision/panorama/laplacianCompositer.hpp index 18e5237e58..51cfa979fa 100644 --- a/src/aliceVision/panorama/laplacianCompositer.hpp +++ b/src/aliceVision/panorama/laplacianCompositer.hpp @@ -7,66 +7,6 @@ namespace aliceVision { -template -bool makeImagePyramidCompatible(image::Image& output, - size_t& out_offset_x, size_t& out_offset_y, - const image::Image& input, - size_t offset_x, size_t offset_y, - size_t num_levels) -{ - - if(num_levels == 0) - { - return false; - } - - double max_scale = 1.0 / pow(2.0, num_levels - 1); - - double low_offset_x = double(offset_x) * max_scale; - double low_offset_y = double(offset_y) * max_scale; - - /*Make sure offset is integer even at the lowest level*/ - double corrected_low_offset_x = floor(low_offset_x); - double corrected_low_offset_y = floor(low_offset_y); - - /*Add some borders on the top and left to make sure mask can be smoothed*/ - corrected_low_offset_x = std::max(0.0, corrected_low_offset_x - 3.0); - corrected_low_offset_y = std::max(0.0, corrected_low_offset_y - 3.0); - - /*Compute offset at largest level*/ - out_offset_x = size_t(corrected_low_offset_x / max_scale); - out_offset_y = size_t(corrected_low_offset_y / max_scale); - - /*Compute difference*/ - double doffset_x = double(offset_x) - double(out_offset_x); - double doffset_y = double(offset_y) - double(out_offset_y); - - /* update size with border update */ - double large_width = double(input.Width()) + doffset_x; - double large_height = double(input.Height()) + doffset_y; - - /* compute size at largest scale */ - double low_width = large_width * max_scale; - double low_height = large_height * max_scale; - - /*Make sure width is integer event at the lowest level*/ - double corrected_low_width = ceil(low_width); - double corrected_low_height = ceil(low_height); - - /*Add some borders on the right and bottom to make sure mask can be smoothed*/ - corrected_low_width = corrected_low_width + 3; - corrected_low_height = corrected_low_height + 3; - - /*Compute size at largest level*/ - size_t width = size_t(corrected_low_width / max_scale); - size_t height = size_t(corrected_low_height / max_scale); - - output = image::Image(width, height, true, T(0.0f)); - output.block(doffset_y, doffset_x, input.Height(), input.Width()) = input; - - return true; -} - class LaplacianCompositer : public Compositer { public: diff --git a/src/aliceVision/panorama/laplacianPyramid.cpp b/src/aliceVision/panorama/laplacianPyramid.cpp index 7e35d76f32..233fb49be0 100644 --- a/src/aliceVision/panorama/laplacianPyramid.cpp +++ b/src/aliceVision/panorama/laplacianPyramid.cpp @@ -329,7 +329,6 @@ bool LaplacianPyramid::apply(const aliceVision::image::Image& int width = source.Width(); int height = source.Height(); - /* Convert mask to alpha layer */ image::Image mask_float(width, height); for(int i = 0; i < height; i++) @@ -435,7 +434,10 @@ bool LaplacianPyramid::apply(const aliceVision::image::Image& convolveGaussian5x5(buf_float, current_weights); downscale(next_weights, buf_float); - merge(current_color, current_weights, l, offset_x, offset_y); + if (!merge(current_color, current_weights, l, offset_x, offset_y)) + { + return false; + } current_color = next_color; current_weights = next_weights; @@ -447,8 +449,12 @@ bool LaplacianPyramid::apply(const aliceVision::image::Image& offset_y /= 2; } - merge(current_color, current_weights, _levels.size() - 1, offset_x, offset_y); + if (!merge(current_color, current_weights, _levels.size() - 1, offset_x, offset_y)) + { + return false; + } pos++; + return true; } @@ -462,13 +468,16 @@ bool LaplacianPyramid::merge(const aliceVision::image::Image& aliceVision::image::Image extractedColor(oimg.Width(), oimg.Height()); aliceVision::image::Image extractedWeight(oimg.Width(), oimg.Height()); - BoundingBox extractBb; extractBb.left = offset_x; extractBb.top = offset_y; extractBb.width = oimg.Width(); extractBb.height = oimg.Height(); + extractBb.clampBottom(img.getHeight() - 1); + BoundingBox inputBb = extractBb; + inputBb.left = 0; + inputBb.top = 0; if (!loopyCachedImageExtract(extractedColor, img, extractBb)) { @@ -480,9 +489,9 @@ bool LaplacianPyramid::merge(const aliceVision::image::Image& return false; } - for(int i = 0; i < oimg.Height(); i++) + for(int i = 0; i < inputBb.height; i++) { - for(int j = 0; j < oimg.Width(); j++) + for(int j = 0; j < inputBb.width; j++) { extractedColor(i, j).r() += oimg(i, j).r() * oweight(i, j); extractedColor(i, j).g() += oimg(i, j).g() * oweight(i, j); @@ -491,14 +500,6 @@ bool LaplacianPyramid::merge(const aliceVision::image::Image& } } - BoundingBox inputBb; - inputBb.left = 0; - inputBb.top = 0; - inputBb.width = extractedColor.Width(); - inputBb.height = extractedColor.Height(); - - - if (!loopyCachedImageAssign(img, extractedColor, extractBb, inputBb)) { return false; } @@ -551,7 +552,7 @@ bool LaplacianPyramid::rebuild(CachedImage& output) int x = 0; int y = 0; - + for (int y = 0; y < _levels[halfLevel].getHeight(); y += processingSize) { for (int x = 0; x < _levels[halfLevel].getWidth(); x += processingSize) @@ -574,6 +575,7 @@ bool LaplacianPyramid::rebuild(CachedImage& output) //If the box has a negative left border, //it is equivalent (with the loop, to shifting at the end) int leftBorder = extractedBb.left - dilatedBb.left; + int topBorder = extractedBb.top - dilatedBb.top; if (dilatedBb.left < 0) { dilatedBb.left = _levels[halfLevel].getWidth() + dilatedBb.left; @@ -582,12 +584,7 @@ bool LaplacianPyramid::rebuild(CachedImage& output) BoundingBox doubleDilatedBb = dilatedBb.doubleSize(); BoundingBox doubleBb = extractedBb.doubleSize(); - - std::cout << extractedBb << std::endl; - std::cout << dilatedBb << std::endl; - std::cout << doubleDilatedBb << std::endl; - std::cout << doubleBb << std::endl; - + aliceVision::image::Image extracted(dilatedBb.width, dilatedBb.height); if (!loopyCachedImageExtract(extracted, _levels[halfLevel], dilatedBb)) @@ -619,9 +616,7 @@ bool LaplacianPyramid::rebuild(CachedImage& output) BoundingBox inputAssigmentBb = doubleBb; inputAssigmentBb.left = 2 * leftBorder; - inputAssigmentBb.top = 0; - - std::cout << inputAssigmentBb << std::endl; + inputAssigmentBb.top = 2 * topBorder; if (!loopyCachedImageAssign(_levels[currentLevel], extractedNext, doubleBb, inputAssigmentBb)) { diff --git a/src/aliceVision/panorama/seams.cpp b/src/aliceVision/panorama/seams.cpp index 065f798b21..1a99163893 100644 --- a/src/aliceVision/panorama/seams.cpp +++ b/src/aliceVision/panorama/seams.cpp @@ -3,6 +3,7 @@ #include "gaussian.hpp" #include "imageOps.hpp" #include "compositer.hpp" +#include "feathering.hpp" namespace aliceVision { @@ -226,208 +227,250 @@ bool WTASeams::append(const aliceVision::image::Image& inputMask, bool HierarchicalGraphcutSeams::setOriginalLabels(CachedImage& labels) { - if (_levelOfInterest == 0) - { - return _graphcut->setOriginalLabels(labels); - } - int scale = pow(2, _levelOfInterest); - int nw = _outputWidth / scale; - int nh = _outputHeight / scale; - - CachedImage smallLabels; - if(!smallLabels.createImage(_cacheManager, nw, nh)) + if (!_graphcuts[0].setOriginalLabels(labels)) { return false; } - int processingSize = 256; - int largeSize = 256 * scale; - - for (int i = 0; i < smallLabels.getHeight(); i+= processingSize) + for (int l = 1; l < _countLevels; l++) { - for (int j = 0; j < smallLabels.getWidth(); j+= processingSize) - { - BoundingBox smallBb; - smallBb.left = j; - smallBb.top = i; - smallBb.width = processingSize; - smallBb.height = processingSize; - smallBb.clampRight(smallLabels.getWidth() - 1); - smallBb.clampBottom(smallLabels.getHeight() - 1); - - BoundingBox smallInputBb; - smallInputBb.left = 0; - smallInputBb.top = 0; - smallInputBb.width = smallBb.width; - smallInputBb.height = smallBb.height; - - - image::Image smallView(smallBb.width, smallBb.height); - - if (!smallLabels.extract(smallView, smallInputBb, smallBb)) - { - return false; - } - BoundingBox largeBb; - largeBb.left = smallBb.left * scale; - largeBb.top = smallBb.top * scale; - largeBb.width = smallBb.width * scale; - largeBb.height = smallBb.height * scale; + CachedImage & largerLabels = _graphcuts[l - 1].getLabels(); + CachedImage & smallerLabels = _graphcuts[l].getLabels(); - BoundingBox largeInputBb; - largeInputBb.left = 0; - largeInputBb.top = 0; - largeInputBb.width = largeBb.width; - largeInputBb.height = largeBb.height; + int processingSize = 256; - image::Image largeView(largeBb.width, largeBb.height); - if (!labels.extract(largeView, largeInputBb, largeBb)) - { - return false; - } - - for (int y = 0; y < smallBb.height; y++) + for (int i = 0; i < smallerLabels.getHeight(); i+= processingSize) + { + for (int j = 0; j < smallerLabels.getWidth(); j+= processingSize) { - for (int x = 0; x < smallBb.width; x++) + BoundingBox smallBb; + smallBb.left = j; + smallBb.top = i; + smallBb.width = processingSize; + smallBb.height = processingSize; + smallBb.clampRight(smallerLabels.getWidth() - 1); + smallBb.clampBottom(smallerLabels.getHeight() - 1); + + BoundingBox smallInputBb; + smallInputBb.left = 0; + smallInputBb.top = 0; + smallInputBb.width = smallBb.width; + smallInputBb.height = smallBb.height; + + BoundingBox largeBb = smallBb.doubleSize(); + + BoundingBox largeInputBb; + largeInputBb.left = 0; + largeInputBb.top = 0; + largeInputBb.width = largeBb.width; + largeInputBb.height = largeBb.height; + + image::Image largeView(largeBb.width, largeBb.height); + if (!largerLabels.extract(largeView, largeInputBb, largeBb)) { - smallView(y, x) = largeView(y * scale, x * scale); + return false; } - } - if (!smallLabels.assign(smallView, smallInputBb, smallBb)) - { - return false; + image::Image smallView(smallBb.width, smallBb.height); + downscale(smallView, largeView); + + if (!smallerLabels.assign(smallView, smallInputBb, smallBb)) + { + return false; + } } } } - - return _graphcut->setOriginalLabels(smallLabels); + + return true; } bool HierarchicalGraphcutSeams::append(const aliceVision::image::Image& input, const aliceVision::image::Image& inputMask, IndexT currentIndex, size_t offsetX, size_t offsetY) { - image::Image resizedColor; - image::Image resizedMask; - - int scale = pow(2, _levelOfInterest); - int levelOffsetX = offsetX / scale; - int levelOffsetY = offsetY / scale; - - if (!downscaleByPowerOfTwo(resizedColor, resizedMask, input, inputMask, _levelOfInterest)) + // Make sure input is compatible with pyramid processing + size_t newOffsetX, newOffsetY; + aliceVision::image::Image potImage; + aliceVision::image::Image potMask; + makeImagePyramidCompatible(potImage, newOffsetX, newOffsetY, input, offsetX, offsetY, _countLevels); + makeImagePyramidCompatible(potMask, newOffsetX, newOffsetY, inputMask, offsetX, offsetY, _countLevels); + + // Fill Color images masked parts with fake but coherent info + aliceVision::image::Image feathered; + if (!feathering(feathered, potImage, potMask)) { return false; } - BoundingBox bb; - bb.left = 0; - bb.top = 0; - bb.width = resizedColor.Width(); - bb.height = resizedColor.Height(); + int width = feathered.Width(); + int height = feathered.Height(); - CachedImage destColor; - if (!destColor.createImage(_cacheManager, resizedColor.Width(), resizedColor.Height())) + for (int level = 0; level < _countLevels; level++) { - return false; - } + BoundingBox bb; + bb.left = 0; + bb.top = 0; + bb.width = width; + bb.height = height; + + //Create cached image and assign image + CachedImage destColor; + if (!destColor.createImage(_cacheManager, width, height)) + { + return false; + } - if (!destColor.fill(image::RGBfColor(0.0f))) - { - return false; - } + if (!destColor.fill(image::RGBfColor(0.0f))) + { + return false; + } - if (!destColor.assign(resizedColor, bb, bb)) - { - return false; - } + if (!destColor.assign(feathered, bb, bb)) + { + return false; + } + //Create cached mask and assign content + CachedImage destMask; + if (!destMask.createImage(_cacheManager, width, height)) + { + return false; + } - CachedImage destMask; - if (!destMask.createImage(_cacheManager, resizedMask.Width(), resizedMask.Height())) - { - return false; - } + if (!destMask.fill(0)) + { + return false; + } - if (!destMask.fill(0)) - { - return false; - } + if (!destMask.assign(potMask, bb, bb)) + { + return false; + } - if (!destMask.assign(resizedMask, bb, bb)) - { - return false; + //Assign content to graphcut + if (!_graphcuts[level].append(destColor, destMask, currentIndex, newOffsetX, newOffsetY)) + { + return false; + } + + if (level == _countLevels - 1) + { + continue; + } + + //Resize for next level + newOffsetX = newOffsetX / 2; + newOffsetY = newOffsetY / 2; + int newWidth = int(ceil(float(width) / 2.0f)); + int newHeight = int(ceil(float(height) / 2.0f)); + + aliceVision::image::Image nextImage(newWidth, newHeight); + aliceVision::image::Image nextMask(newWidth, newHeight); + aliceVision::image::Image buf(width, height); + + //Convolve + divide + convolveGaussian5x5(buf, feathered); + downscale(nextImage, buf); + + //Just nearest neighboor divide for mask + for(int i = 0; i < nextMask.Height(); i++) + { + int di = i * 2; + + for(int j = 0; j < nextMask.Width(); j++) + { + int dj = j * 2; + + if(potMask(di, dj) && potMask(di, dj + 1) + && potMask(di + 1, dj) && potMask(di + 1, dj + 1)) + { + nextMask(i, j) = 255; + } + else + { + nextMask(i, j) = 0; + } + } + } + + + std::swap(feathered, nextImage); + std::swap(potMask, nextMask); + + width = newWidth; + height = newHeight; } - - return _graphcut->append(destColor, destMask, currentIndex, levelOffsetX, levelOffsetY); + return true; } bool HierarchicalGraphcutSeams::process() { - if(!_graphcut->process()) + for (int level = _countLevels - 1; level >= 0; level--) { - return false; - } - - CachedImage smallLabels = _graphcut->getLabels(); + if(!_graphcuts[level].process()) + { + return false; + } - int scale = pow(2, _levelOfInterest); + if (level == 0) + { + return true; + } + + //Enlarge result of this level to be an initialization for next level + CachedImage & smallLabels = _graphcuts[level].getLabels(); + CachedImage & largeLabels = _graphcuts[level - 1].getLabels(); - int processingSize = 256; - int largeSize = 256 * scale; + const int processingSize = 256; - for (int i = 0; i < smallLabels.getHeight(); i+= processingSize) - { - for (int j = 0; j < smallLabels.getWidth(); j+= processingSize) + for (int i = 0; i < smallLabels.getHeight(); i+= processingSize) { - BoundingBox smallBb; - smallBb.left = j; - smallBb.top = i; - smallBb.width = processingSize; - smallBb.height = processingSize; - smallBb.clampRight(smallLabels.getWidth() - 1); - smallBb.clampBottom(smallLabels.getHeight() - 1); - - BoundingBox smallInputBb; - smallInputBb.left = 0; - smallInputBb.top = 0; - smallInputBb.width = smallBb.width; - smallInputBb.height = smallBb.height; - - - image::Image smallView(smallBb.width, smallBb.height); - if (!smallLabels.extract(smallView, smallInputBb, smallBb)) + for (int j = 0; j < smallLabels.getWidth(); j+= processingSize) { - return false; - } - - BoundingBox largeBb; - largeBb.left = smallBb.left * scale; - largeBb.top = smallBb.top * scale; - largeBb.width = smallBb.width * scale; - largeBb.height = smallBb.height * scale; + BoundingBox smallBb; + smallBb.left = j; + smallBb.top = i; + smallBb.width = processingSize; + smallBb.height = processingSize; + smallBb.clampRight(smallLabels.getWidth() - 1); + smallBb.clampBottom(smallLabels.getHeight() - 1); + + BoundingBox smallInputBb = smallBb; + smallInputBb.left = 0; + smallInputBb.top = 0; + + + image::Image smallView(smallBb.width, smallBb.height); + if (!smallLabels.extract(smallView, smallInputBb, smallBb)) + { + return false; + } - BoundingBox largeInputBb; - largeInputBb.left = 0; - largeInputBb.top = 0; - largeInputBb.width = largeBb.width; - largeInputBb.height = largeBb.height; + BoundingBox largeBb = smallBb.doubleSize(); + BoundingBox largeInputBb = largeBb; + largeInputBb.left = 0; + largeInputBb.top = 0; - image::Image largeView(largeBb.width, largeBb.height); - for (int y = 0; y < largeBb.height; y++) - { - for (int x = 0; x < largeBb.width; x++) + image::Image largeView(largeBb.width, largeBb.height); + for (int y = 0; y < largeBb.height; y++) { - largeView(y, x) = smallView(y / scale, x / scale); + int hy = y / 2; + + for (int x = 0; x < largeBb.width; x++) + { + int hx = x / 2; + largeView(y, x) = smallView(hy, hx); + } } - } - if (!_labels.assign(largeView, largeInputBb, largeBb)) - { - return false; + if (!largeLabels.assign(largeView, largeInputBb, largeBb)) + { + return false; + } } } } @@ -438,19 +481,22 @@ bool HierarchicalGraphcutSeams::process() bool HierarchicalGraphcutSeams::initialize() { - if (!_graphcut->initialize(_cacheManager)) - { - return false; - } + int width = _outputWidth; + int height = _outputHeight; - if(!_labels.createImage(_cacheManager, _outputWidth, _outputHeight)) + for (int i = 0; i < _countLevels; i++) { - return false; - } + GraphcutSeams gcs(width, height); - if(!_labels.fill(UndefinedIndexT)) - { - return false; + if (!gcs.initialize(_cacheManager)) + { + return false; + } + + _graphcuts.push_back(gcs); + + width = int(ceil(float(width) / 2.0f)); + height = int(ceil(float(height) / 2.0f)); } return true; diff --git a/src/aliceVision/panorama/seams.hpp b/src/aliceVision/panorama/seams.hpp index 30a5088577..5b422bd306 100644 --- a/src/aliceVision/panorama/seams.hpp +++ b/src/aliceVision/panorama/seams.hpp @@ -49,16 +49,12 @@ class WTASeams class HierarchicalGraphcutSeams { public: - HierarchicalGraphcutSeams(image::TileCacheManager::shared_ptr cacheManager, size_t outputWidth, size_t outputHeight, size_t levelOfInterest) + HierarchicalGraphcutSeams(image::TileCacheManager::shared_ptr cacheManager, size_t outputWidth, size_t outputHeight, size_t countLevels) : _cacheManager(cacheManager) , _outputWidth(outputWidth) , _outputHeight(outputHeight) - , _levelOfInterest(levelOfInterest) + , _countLevels(countLevels) { - double scale = 1.0 / pow(2.0, levelOfInterest); - size_t width = size_t(floor(double(outputWidth) * scale)); - size_t height = size_t(floor(double(outputHeight) * scale)); - _graphcut = std::unique_ptr(new GraphcutSeams(width, height)); } virtual ~HierarchicalGraphcutSeams() = default; @@ -67,11 +63,6 @@ class HierarchicalGraphcutSeams bool setOriginalLabels(CachedImage& labels); - void setMaximalDistance(int distance) - { - _graphcut->setMaximalDistance(distance); - } - virtual bool append(const aliceVision::image::Image& input, const aliceVision::image::Image& inputMask, IndexT currentIndex, size_t offset_x, size_t offset_y); @@ -80,15 +71,14 @@ class HierarchicalGraphcutSeams CachedImage& getLabels() { - return _labels; + return _graphcuts[0].getLabels(); } private: - std::unique_ptr _graphcut; + std::vector _graphcuts; image::TileCacheManager::shared_ptr _cacheManager; - CachedImage _labels; - size_t _levelOfInterest; + size_t _countLevels; size_t _outputWidth; size_t _outputHeight; }; From ac20d6ea4e33f8ce066e8908479a64e27d3607cb Mon Sep 17 00:00:00 2001 From: Fabien Servant Date: Mon, 19 Oct 2020 14:23:07 +0200 Subject: [PATCH 19/79] [panorama] graphcut debug --- src/aliceVision/panorama/graphcut.hpp | 15 +-- src/aliceVision/panorama/seams.cpp | 7 ++ .../pipeline/main_panoramaCompositing.cpp | 92 +++++++++---------- 3 files changed, 57 insertions(+), 57 deletions(-) diff --git a/src/aliceVision/panorama/graphcut.hpp b/src/aliceVision/panorama/graphcut.hpp index c454b519e5..a9e3a1aa04 100644 --- a/src/aliceVision/panorama/graphcut.hpp +++ b/src/aliceVision/panorama/graphcut.hpp @@ -507,6 +507,12 @@ class GraphcutSeams continue; } + float dist = sqrt(float(distanceMap(y, x))); + if (dist > _maximal_distance_change) + { + continue; + } + PixelInfo & pix = graphCutInput(y, x); if (pix.size() == 0) @@ -561,9 +567,8 @@ class GraphcutSeams for (int i = 0; i < 10; i++) { - std::cout << "**************************" << std::endl; + ALICEVISION_LOG_INFO("GraphCut processing iteration #" << i); // For each possible label, try to extends its domination on the label's world - bool hasChange = false; int pos = 0; @@ -577,8 +582,6 @@ class GraphcutSeams if (costs[info.first] != cost) { - std::cout << costs[info.first] << " ----> " << cost << std::endl; - costs[info.first] = cost; hasChange = true; } @@ -590,9 +593,9 @@ class GraphcutSeams } } - char filename[512]; + /*char filename[512]; sprintf(filename, "/home/mmoc/test%d.exr", _outputWidth); - _labels.writeImage(filename); + _labels.writeImage(filename);*/ return true; } diff --git a/src/aliceVision/panorama/seams.cpp b/src/aliceVision/panorama/seams.cpp index 1a99163893..a20813d0bf 100644 --- a/src/aliceVision/panorama/seams.cpp +++ b/src/aliceVision/panorama/seams.cpp @@ -411,6 +411,13 @@ bool HierarchicalGraphcutSeams::process() { for (int level = _countLevels - 1; level >= 0; level--) { + ALICEVISION_LOG_INFO("Hierachical graphcut processing level #" << level); + + if (level < _countLevels - 1) + { + _graphcuts[level].setMaximalDistance(10); + } + if(!_graphcuts[level].process()) { return false; diff --git a/src/software/pipeline/main_panoramaCompositing.cpp b/src/software/pipeline/main_panoramaCompositing.cpp index f3e95ce985..0d2945f5a5 100644 --- a/src/software/pipeline/main_panoramaCompositing.cpp +++ b/src/software/pipeline/main_panoramaCompositing.cpp @@ -137,70 +137,60 @@ bool computeGCLabels(CachedImage & labels, image::TileCacheManager::shar double ratio = double(panoramaSize.first) / double(min_width_for_graphcut); if (ratio > 1.0) { initial_level = int(ceil(log2(ratio))); - } + } - for (int l = initial_level; l>= 0; l--) - { - - HierarchicalGraphcutSeams seams(cacheManager, panoramaSize.first, panoramaSize.second, l); + HierarchicalGraphcutSeams seams(cacheManager, panoramaSize.first, panoramaSize.second, initial_level + 1); - - if (!seams.initialize()) - { - return false; - } + if (!seams.initialize()) + { + return false; + } - if (!seams.setOriginalLabels(labels)) - { - return false; - } + if (!seams.setOriginalLabels(labels)) + { + return false; + } - if (l != initial_level) + for (const auto& viewIt : sfmData.getViews()) + { + if(!sfmData.isPoseAndIntrinsicDefined(viewIt.second.get())) { - seams.setMaximalDistance(100); + // skip unreconstructed views + continue; } - for (const auto& viewIt : sfmData.getViews()) - { - if(!sfmData.isPoseAndIntrinsicDefined(viewIt.second.get())) - { - // skip unreconstructed views - continue; - } - - // Load mask - const std::string maskPath = (fs::path(inputPath) / (std::to_string(viewIt.first) + "_mask.exr")).string(); - ALICEVISION_LOG_INFO("Load mask with path " << maskPath); - image::Image mask; - image::readImage(maskPath, mask, image::EImageColorSpace::NO_CONVERSION); - - // Load Color - const std::string colorsPath = (fs::path(inputPath) / (std::to_string(viewIt.first) + ".exr")).string(); - ALICEVISION_LOG_INFO("Load colors with path " << colorsPath); - image::Image colors; - image::readImage(colorsPath, colors, image::EImageColorSpace::NO_CONVERSION); + // Load mask + const std::string maskPath = (fs::path(inputPath) / (std::to_string(viewIt.first) + "_mask.exr")).string(); + ALICEVISION_LOG_INFO("Load mask with path " << maskPath); + image::Image mask; + image::readImage(maskPath, mask, image::EImageColorSpace::NO_CONVERSION); - // Get offset - oiio::ParamValueList metadata = image::readImageMetadata(maskPath); - const std::size_t offsetX = metadata.find("AliceVision:offsetX")->get_int(); - const std::size_t offsetY = metadata.find("AliceVision:offsetY")->get_int(); + // Load Color + const std::string colorsPath = (fs::path(inputPath) / (std::to_string(viewIt.first) + ".exr")).string(); + ALICEVISION_LOG_INFO("Load colors with path " << colorsPath); + image::Image colors; + image::readImage(colorsPath, colors, image::EImageColorSpace::NO_CONVERSION); + // Get offset + oiio::ParamValueList metadata = image::readImageMetadata(maskPath); + const std::size_t offsetX = metadata.find("AliceVision:offsetX")->get_int(); + const std::size_t offsetY = metadata.find("AliceVision:offsetY")->get_int(); - // Append to graph cut - if (!seams.append(colors, mask, viewIt.first, offsetX, offsetY)) - { - return false; - } - } - if (!seams.process()) + // Append to graph cut + if (!seams.append(colors, mask, viewIt.first, offsetX, offsetY)) { return false; } + } - labels = seams.getLabels(); + if (!seams.process()) + { + return false; } + labels = seams.getLabels(); + return true; } @@ -352,16 +342,15 @@ int aliceVision_main(int argc, char** argv) return EXIT_FAILURE; } - labels.writeImage("/home/mmoc/labels_wta.exr"); + //labels.writeImage("/home/mmoc/labels_wta.exr"); - /* if (!computeGCLabels(labels, cacheManager, sfmData, warpingFolder, panoramaSize)) + if (!computeGCLabels(labels, cacheManager, sfmData, warpingFolder, panoramaSize)) { ALICEVISION_LOG_ERROR("Error computing graph cut labels"); return EXIT_FAILURE; } - labels.writeImage("/home/mmoc/labels_gc.exr"); -*/ + //labels.writeImage("/home/mmoc/labels_gc.exr"); //Get a list of views ordered by their image scale std::vector> viewOrderedByScale; @@ -400,6 +389,7 @@ int aliceVision_main(int argc, char** argv) continue; } pos++; + // Load image and convert it to linear colorspace const std::string imagePath = (fs::path(warpingFolder) / (std::to_string(viewId) + ".exr")).string(); From df5c5a568cfe6bbec8dc684bcd7005441310e3fd Mon Sep 17 00:00:00 2001 From: Fabien Servant Date: Tue, 20 Oct 2020 11:10:32 +0200 Subject: [PATCH 20/79] [panorama] make things work for odd sizes --- src/aliceVision/panorama/alphaCompositer.hpp | 2 +- src/aliceVision/panorama/compositer.hpp | 2 +- src/aliceVision/panorama/coordinatesMap.cpp | 4 + src/aliceVision/panorama/feathering.cpp | 43 +++++++ src/aliceVision/panorama/imageOps.hpp | 63 ++++++++-- .../panorama/laplacianCompositer.hpp | 9 +- src/aliceVision/panorama/laplacianPyramid.cpp | 117 ++++++++++++------ src/aliceVision/panorama/seams.cpp | 26 +++- .../pipeline/main_panoramaCompositing.cpp | 38 ++---- .../pipeline/main_panoramaWarping.cpp | 34 ++--- 10 files changed, 233 insertions(+), 105 deletions(-) diff --git a/src/aliceVision/panorama/alphaCompositer.hpp b/src/aliceVision/panorama/alphaCompositer.hpp index c895c05eec..8a5bafdf6a 100644 --- a/src/aliceVision/panorama/alphaCompositer.hpp +++ b/src/aliceVision/panorama/alphaCompositer.hpp @@ -16,7 +16,7 @@ class AlphaCompositer : public Compositer virtual bool append(const aliceVision::image::Image& color, const aliceVision::image::Image& inputMask, const aliceVision::image::Image& inputWeights, - int offset_x, int offset_y, const BoundingBox & contentBox) + int offset_x, int offset_y) { aliceVision::image::Image masked(color.Width(), color.Height()); diff --git a/src/aliceVision/panorama/compositer.hpp b/src/aliceVision/panorama/compositer.hpp index 3f2a2d1b12..7c5e0212ba 100644 --- a/src/aliceVision/panorama/compositer.hpp +++ b/src/aliceVision/panorama/compositer.hpp @@ -22,7 +22,7 @@ class Compositer virtual bool append(const aliceVision::image::Image& color, const aliceVision::image::Image& inputMask, const aliceVision::image::Image& inputWeights, - int offset_x, int offset_y, const BoundingBox & contentBox) + int offset_x, int offset_y) { aliceVision::image::Image masked(color.Width(), color.Height()); diff --git a/src/aliceVision/panorama/coordinatesMap.cpp b/src/aliceVision/panorama/coordinatesMap.cpp index d9cd2af34b..f4988c72cd 100644 --- a/src/aliceVision/panorama/coordinatesMap.cpp +++ b/src/aliceVision/panorama/coordinatesMap.cpp @@ -22,6 +22,10 @@ bool CoordinatesMap::build(const std::pair& panoramaSize, const geomet { int cy = y + coarseBbox.top; + if (cy < 0 || cy >= panoramaSize.second) + { + continue; + } for(int x = 0; x < coarseBbox.width; x++) { diff --git a/src/aliceVision/panorama/feathering.cpp b/src/aliceVision/panorama/feathering.cpp index ab49bd8a3c..fba6f98b34 100644 --- a/src/aliceVision/panorama/feathering.cpp +++ b/src/aliceVision/panorama/feathering.cpp @@ -84,6 +84,49 @@ bool feathering(aliceVision::image::Image& output, lvl++; } + //Now we want to make sure we have no masked pixel with undefined color + //So we compute the mean of all valid pixels, and set the invalid pixels to this value + image::Image & lastImage = feathering[feathering.size() - 1]; + image::Image & lastMask = feathering_mask[feathering_mask.size() - 1]; + image::RGBfColor sum(0.0f); + int count = 0; + for (int y = 0; y < height; y++) + { + for (int x = 0; x < width; x++) + { + if (lastMask(y, x)) + { + sum.r() += lastImage(y, x).r(); + sum.g() += lastImage(y, x).g(); + sum.b() += lastImage(y, x).b(); + count++; + } + } + } + + if (count > 0) + { + image::RGBfColor mean; + mean.r() = sum.r() / float(count); + mean.g() = sum.g() / float(count); + mean.b() = sum.b() / float(count); + + for (int y = 0; y < height; y++) + { + for (int x = 0; x < width; x++) + { + if (!lastMask(y, x)) + { + lastImage(y, x) = mean; + lastMask(y, x) = 255; + } + } + } + } + + + //Now, level by level, we fill masked pixel with the estimated value from + //The lower level. for(int lvl = feathering.size() - 2; lvl >= 0; lvl--) { diff --git a/src/aliceVision/panorama/imageOps.hpp b/src/aliceVision/panorama/imageOps.hpp index bb0dee4e7f..5eafe22777 100644 --- a/src/aliceVision/panorama/imageOps.hpp +++ b/src/aliceVision/panorama/imageOps.hpp @@ -9,13 +9,9 @@ namespace aliceVision template bool downscale(aliceVision::image::Image& outputColor, const aliceVision::image::Image& inputColor) { - - size_t output_width = inputColor.Width() / 2; - size_t output_height = inputColor.Height() / 2; - - for(int i = 0; i < output_height; i++) + for(int i = 0; i < outputColor.Height(); i++) { - for(int j = 0; j < output_width; j++) + for(int j = 0; j < outputColor.Width(); j++) { outputColor(i, j) = inputColor(i * 2, j * 2); } @@ -30,13 +26,14 @@ bool upscale(aliceVision::image::Image& outputColor, const aliceVision::image size_t width = inputColor.Width(); size_t height = inputColor.Height(); + size_t dwidth = outputColor.Width(); + size_t dheight = outputColor.Height(); - for(int i = 0; i < height; i++) + for(int i = 0; i < height - 1; i++) { - int di = i * 2; - for(int j = 0; j < width; j++) + for(int j = 0; j < width - 1; j++) { int dj = j * 2; @@ -47,6 +44,54 @@ bool upscale(aliceVision::image::Image& outputColor, const aliceVision::image } } + for (int i = 0; i < height; i++) + { + int j = width - 1; + int di = i * 2; + int dj = j * 2; + + outputColor(di, dj) = T(); + + if (dj < dwidth - 1) + { + outputColor(di, dj + 1) = T(); + } + + if (di < dheight - 1) + { + outputColor(di + 1, dj) = T(); + + if (dj < dwidth - 1) + { + outputColor(di + 1, dj + 1) = inputColor(i, j); + } + } + } + + for (int j = 0; j < width; j++) + { + int i = height - 1; + int di = i * 2; + int dj = j * 2; + + outputColor(di, dj) = T(); + + if (dj < dwidth - 1) + { + outputColor(di, dj + 1) = T(); + } + + if (di < dheight - 1) + { + outputColor(di + 1, dj) = T(); + + if (dj < dwidth - 1) + { + outputColor(di + 1, dj + 1) = inputColor(i, j); + } + } + } + return true; } diff --git a/src/aliceVision/panorama/laplacianCompositer.hpp b/src/aliceVision/panorama/laplacianCompositer.hpp index 51cfa979fa..1c714ff624 100644 --- a/src/aliceVision/panorama/laplacianCompositer.hpp +++ b/src/aliceVision/panorama/laplacianCompositer.hpp @@ -47,9 +47,10 @@ class LaplacianCompositer : public Compositer virtual bool append(const aliceVision::image::Image& color, const aliceVision::image::Image& inputMask, const aliceVision::image::Image& inputWeights, - size_t offset_x, size_t offset_y, const BoundingBox & contentBox) + size_t offset_x, size_t offset_y) { - size_t optimalScale = getOptimalScale(contentBox.width, contentBox.height); + size_t optimalScale = getOptimalScale(color.Width(), color.Height()); + std::cout << optimalScale << std::endl; if(optimalScale < _bands) { ALICEVISION_LOG_ERROR("Decreasing scale !"); @@ -58,11 +59,11 @@ class LaplacianCompositer : public Compositer //If the input scale is more important than previously processed, // The pyramid must be deepened accordingly - if(optimalScale > _bands) + /*if(optimalScale > _bands) { _bands = optimalScale; _pyramidPanorama.augment(_cacheManager, _bands); - } + }*/ // Make sure input is compatible with pyramid processing size_t new_offset_x, new_offset_y; diff --git a/src/aliceVision/panorama/laplacianPyramid.cpp b/src/aliceVision/panorama/laplacianPyramid.cpp index 233fb49be0..621e347d96 100644 --- a/src/aliceVision/panorama/laplacianPyramid.cpp +++ b/src/aliceVision/panorama/laplacianPyramid.cpp @@ -53,8 +53,8 @@ bool LaplacianPyramid::initialize(image::TileCacheManager::shared_ptr & cacheMan _levels.push_back(color); _weights.push_back(weights); - height /= 2; - width /= 2; + height = int(ceil(float(height) / 2.0f)); + width = int(ceil(float(width) / 2.0f)); } return true; @@ -62,8 +62,6 @@ bool LaplacianPyramid::initialize(image::TileCacheManager::shared_ptr & cacheMan bool LaplacianPyramid::augment(image::TileCacheManager::shared_ptr & cacheManager, size_t newMaxLevels) { - //return true; - ALICEVISION_LOG_INFO("augment number of levels to " << newMaxLevels); if(newMaxLevels <= _levels.size()) @@ -326,6 +324,9 @@ bool LaplacianPyramid::apply(const aliceVision::image::Image& const aliceVision::image::Image& mask, const aliceVision::image::Image& weights, size_t offset_x, size_t offset_y) { + //We assume the input source has been feathered + //and resized to be a simili power of 2 for the needed scales. + int width = source.Width(); int height = source.Height(); @@ -474,6 +475,7 @@ bool LaplacianPyramid::merge(const aliceVision::image::Image& extractBb.width = oimg.Width(); extractBb.height = oimg.Height(); extractBb.clampBottom(img.getHeight() - 1); + BoundingBox inputBb = extractBb; inputBb.left = 0; @@ -488,15 +490,35 @@ bool LaplacianPyramid::merge(const aliceVision::image::Image& { return false; } - - for(int i = 0; i < inputBb.height; i++) + + bool special = false; + if (img.getWidth() % 2) { - for(int j = 0; j < inputBb.width; j++) - { - extractedColor(i, j).r() += oimg(i, j).r() * oweight(i, j); - extractedColor(i, j).g() += oimg(i, j).g() * oweight(i, j); - extractedColor(i, j).b() += oimg(i, j).b() * oweight(i, j); - extractedWeight(i, j) += oweight(i, j); + special = true; + } + + for(int y = 0; y < inputBb.height; y++) + { + int vx = 0; + for(int x = 0; x < inputBb.width && vx < inputBb.width; x++) + { + //Special process for levels with a width non divided by 2 + //To account for the upscale difference, we "add" a black column + if (special) + { + int posX = offset_x + x; + if (posX == img.getWidth() - 1) + { + vx++; + } + } + + extractedColor(y, x).r() += oimg(y, vx).r() * oweight(y, vx); + extractedColor(y, x).g() += oimg(y, vx).g() * oweight(y, vx); + extractedColor(y, x).b() += oimg(y, vx).b() * oweight(y, vx); + extractedWeight(y, x) += oweight(y, vx); + + vx++; } } @@ -513,8 +535,6 @@ bool LaplacianPyramid::merge(const aliceVision::image::Image& bool LaplacianPyramid::rebuild(CachedImage& output) { - - // We first want to compute the final pixels mean for(int l = 0; l < _levels.size(); l++) { @@ -538,7 +558,7 @@ bool LaplacianPyramid::rebuild(CachedImage& output) } - + removeNegativeValues(_levels[_levels.size() - 1]); for(int l = _levels.size() - 2; l >= 0; l--) @@ -551,22 +571,35 @@ bool LaplacianPyramid::rebuild(CachedImage& output) int x = 0; int y = 0; - - for (int y = 0; y < _levels[halfLevel].getHeight(); y += processingSize) + bool specialBorder = false; + if (_levels[halfLevel].getWidth() % 2) { - for (int x = 0; x < _levels[halfLevel].getWidth(); x += processingSize) + specialBorder = true; + } + + for (int py = 0; py < _levels[halfLevel].getHeight(); py += processingSize) + { + for (int px = 0; px < _levels[halfLevel].getWidth(); px += processingSize) { //Estimate the Bounding box //Make sure we are not processing outside of the image bounding box BoundingBox extractedBb; - extractedBb.left = x; - extractedBb.top = y; + extractedBb.left = px; + extractedBb.top = py; extractedBb.width = processingSize; extractedBb.height = processingSize; + extractedBb.clampLeft(); + extractedBb.clampTop(); extractedBb.clampRight(_levels[halfLevel].getWidth() - 1); extractedBb.clampBottom(_levels[halfLevel].getHeight() - 1); + BoundingBox doubleBb = extractedBb.doubleSize(); + doubleBb.clampLeft(); + doubleBb.clampTop(); + doubleBb.clampRight(_levels[currentLevel].getWidth() - 1); + doubleBb.clampBottom(_levels[currentLevel].getHeight() - 1); + //Add borders to this bounding box for filtering BoundingBox dilatedBb = extractedBb.dilate(borderSize); dilatedBb.clampTop(); @@ -579,12 +612,7 @@ bool LaplacianPyramid::rebuild(CachedImage& output) if (dilatedBb.left < 0) { dilatedBb.left = _levels[halfLevel].getWidth() + dilatedBb.left; - } - - BoundingBox doubleDilatedBb = dilatedBb.doubleSize(); - BoundingBox doubleBb = extractedBb.doubleSize(); - - + } aliceVision::image::Image extracted(dilatedBb.width, dilatedBb.height); if (!loopyCachedImageExtract(extracted, _levels[halfLevel], dilatedBb)) @@ -592,42 +620,55 @@ bool LaplacianPyramid::rebuild(CachedImage& output) return false; } - aliceVision::image::Image extractedNext(doubleDilatedBb.width, doubleDilatedBb.height); - if (!loopyCachedImageExtract(extractedNext, _levels[currentLevel], doubleDilatedBb)) + aliceVision::image::Image extractedNext(doubleBb.width, doubleBb.height); + if (!loopyCachedImageExtract(extractedNext, _levels[currentLevel], doubleBb)) { return false; } - aliceVision::image::Image buf(doubleDilatedBb.width, doubleDilatedBb.height); - aliceVision::image::Image buf2(doubleDilatedBb.width, doubleDilatedBb.height); + aliceVision::image::Image buf(dilatedBb.width * 2, dilatedBb.height * 2); + aliceVision::image::Image buf2(dilatedBb.width * 2, dilatedBb.height * 2); upscale(buf, extracted); convolveGaussian5x5(buf2, buf, false); - for(int i = 0; i < buf2.Height(); i++) + for(int y = 0; y < buf2.Height(); y++) { - for(int j = 0; j < buf2.Width(); j++) + for(int x = 0; x < buf2.Width(); x++) { - buf2(i, j) *= 4.0f; + buf2(y, x) *= 4.0f; + } + } + + int shiftY = topBorder * 2; + int shiftX = leftBorder * 2; + for (int y = 0; y < doubleBb.height; y++) + { + for (int x = 0; x < doubleBb.width; x++) + { + extractedNext(y, x) += buf2(y + shiftY, x + shiftX); } } - - addition(extractedNext, extractedNext, buf2); BoundingBox inputAssigmentBb = doubleBb; - inputAssigmentBb.left = 2 * leftBorder; - inputAssigmentBb.top = 2 * topBorder; - + inputAssigmentBb.left = 0; + inputAssigmentBb.top = 0; if (!loopyCachedImageAssign(_levels[currentLevel], extractedNext, doubleBb, inputAssigmentBb)) { + std::cout << "failed assign" << std::endl; return false; } } } removeNegativeValues(_levels[currentLevel]); + + } + + + for(int i = 0; i < output.getTiles().size(); i++) { diff --git a/src/aliceVision/panorama/seams.cpp b/src/aliceVision/panorama/seams.cpp index a20813d0bf..6272e7a932 100644 --- a/src/aliceVision/panorama/seams.cpp +++ b/src/aliceVision/panorama/seams.cpp @@ -260,6 +260,8 @@ bool HierarchicalGraphcutSeams::setOriginalLabels(CachedImage& labels) smallInputBb.height = smallBb.height; BoundingBox largeBb = smallBb.doubleSize(); + largeBb.clampRight(largerLabels.getWidth() - 1); + largeBb.clampBottom(largerLabels.getHeight() - 1); BoundingBox largeInputBb; largeInputBb.left = 0; @@ -384,8 +386,24 @@ bool HierarchicalGraphcutSeams::append(const aliceVision::image::Imageget_int(); const std::size_t offsetY = metadata.find("AliceVision:offsetY")->get_int(); - const std::size_t contentX = metadata.find("AliceVision:contentX")->get_int(); - const std::size_t contentY = metadata.find("AliceVision:contentY")->get_int(); - const std::size_t contentW = metadata.find("AliceVision:contentW")->get_int(); - const std::size_t contentH = metadata.find("AliceVision:contentH")->get_int(); BoundingBox bb; bb.left = offsetX + contentX; @@ -76,7 +72,7 @@ bool buildMap(BoundingBoxMap & map, const sfmData::SfMData& sfmData, const std:: { return false; } - } + }*/ return true; } @@ -136,7 +132,7 @@ bool computeGCLabels(CachedImage & labels, image::TileCacheManager::shar int min_width_for_graphcut = 1000; double ratio = double(panoramaSize.first) / double(min_width_for_graphcut); if (ratio > 1.0) { - initial_level = int(ceil(log2(ratio))); + initial_level = int(floor(log2(ratio))); } HierarchicalGraphcutSeams seams(cacheManager, panoramaSize.first, panoramaSize.second, initial_level + 1); @@ -320,7 +316,7 @@ int aliceVision_main(int argc, char** argv) // Configure the cache manager memory cacheManager->setInCoreMaxObjectCount(1000); - LaplacianCompositer compositer(cacheManager, panoramaSize.first, panoramaSize.second, 1); + LaplacianCompositer compositer(cacheManager, panoramaSize.first, panoramaSize.second, 5); if (!compositer.initialize()) { @@ -342,15 +338,15 @@ int aliceVision_main(int argc, char** argv) return EXIT_FAILURE; } - //labels.writeImage("/home/mmoc/labels_wta.exr"); + labels.writeImage("/home/mmoc/labels_wta.exr"); - if (!computeGCLabels(labels, cacheManager, sfmData, warpingFolder, panoramaSize)) + /*if (!computeGCLabels(labels, cacheManager, sfmData, warpingFolder, panoramaSize)) { ALICEVISION_LOG_ERROR("Error computing graph cut labels"); return EXIT_FAILURE; } - //labels.writeImage("/home/mmoc/labels_gc.exr"); + labels.writeImage("/home/mmoc/labels_gc.exr");*/ //Get a list of views ordered by their image scale std::vector> viewOrderedByScale; @@ -358,13 +354,7 @@ int aliceVision_main(int argc, char** argv) std::map>> mapViewsScale; for(const auto& it : sfmData.getViews()) { - const std::string maskPath = (fs::path(warpingFolder) / (std::to_string(it.first) + "_mask.exr")).string(); - oiio::ParamValueList metadata = image::readImageMetadata(maskPath); - - const std::size_t contentW = metadata.find("AliceVision:contentW")->get_int(); - const std::size_t contentH = metadata.find("AliceVision:contentH")->get_int(); - - size_t scale = compositer.getOptimalScale(contentW, contentH); + size_t scale = compositer.getOptimalScale(it.second->getWidth(), it.second->getHeight()); mapViewsScale[scale].push_back(it.second); } for (auto scaledList : mapViewsScale) @@ -400,11 +390,6 @@ int aliceVision_main(int argc, char** argv) oiio::ParamValueList metadata = image::readImageMetadata(imagePath); const std::size_t offsetX = metadata.find("AliceVision:offsetX")->get_int(); const std::size_t offsetY = metadata.find("AliceVision:offsetY")->get_int(); - const std::size_t contentX = metadata.find("AliceVision:contentX")->get_int(); - const std::size_t contentY = metadata.find("AliceVision:contentY")->get_int(); - const std::size_t contentW = metadata.find("AliceVision:contentW")->get_int(); - const std::size_t contentH = metadata.find("AliceVision:contentH")->get_int(); - // Load mask const std::string maskPath = (fs::path(warpingFolder) / (std::to_string(viewId) + "_mask.exr")).string(); @@ -419,11 +404,6 @@ int aliceVision_main(int argc, char** argv) image::Image weights; image::readImage(weightsPath, weights, image::EImageColorSpace::NO_CONVERSION);*/ - BoundingBox imageContent; - imageContent.left = contentX; - imageContent.top = contentY; - imageContent.width = contentW; - imageContent.height = contentH; image::Image seams(mask.Width(), mask.Height()); if (!getMaskFromLabels(seams, labels, viewId, offsetX, offsetY)) @@ -433,7 +413,7 @@ int aliceVision_main(int argc, char** argv) } std::cout << pos << std::endl; - if (!compositer.append(source, mask, seams, offsetX, offsetY, imageContent)) + if (!compositer.append(source, mask, seams, offsetX, offsetY)) { return EXIT_FAILURE; } diff --git a/src/software/pipeline/main_panoramaWarping.cpp b/src/software/pipeline/main_panoramaWarping.cpp index b9ca83fccf..e6aeb3aade 100644 --- a/src/software/pipeline/main_panoramaWarping.cpp +++ b/src/software/pipeline/main_panoramaWarping.cpp @@ -329,14 +329,10 @@ int aliceVision_main(int argc, char** argv) globalBbox.left -= panoramaSize.first; } - // Update bounding box - BoundingBox snappedGlobalBbox; + globalBbox.width = std::min(globalBbox.width, panoramaSize.first); - // Once again, snap to grid - snappedGlobalBbox = globalBbox; - //snappedGlobalBbox.snapToGrid(tileSize); - if (snappedGlobalBbox.width <= 0 || snappedGlobalBbox.height <= 0) continue; + globalBbox.height = std::min(globalBbox.height, panoramaSize.second); // Load image and convert it to linear colorspace std::string imagePath = view.getImagePath(); @@ -346,12 +342,8 @@ int aliceVision_main(int argc, char** argv) // Load metadata and update for output oiio::ParamValueList metadata = image::readImageMetadata(imagePath); - metadata.push_back(oiio::ParamValue("AliceVision:offsetX", snappedGlobalBbox.left)); - metadata.push_back(oiio::ParamValue("AliceVision:offsetY", snappedGlobalBbox.top)); - metadata.push_back(oiio::ParamValue("AliceVision:contentX", globalBbox.left - snappedGlobalBbox.left)); - metadata.push_back(oiio::ParamValue("AliceVision:contentY", globalBbox.top - snappedGlobalBbox.top)); - metadata.push_back(oiio::ParamValue("AliceVision:contentW", globalBbox.width)); - metadata.push_back(oiio::ParamValue("AliceVision:contentH", globalBbox.height)); + metadata.push_back(oiio::ParamValue("AliceVision:offsetX", globalBbox.left)); + metadata.push_back(oiio::ParamValue("AliceVision:offsetY", globalBbox.top)); metadata.push_back(oiio::ParamValue("AliceVision:panoramaWidth", panoramaSize.first)); metadata.push_back(oiio::ParamValue("AliceVision:panoramaHeight", panoramaSize.second)); metadata.push_back(oiio::ParamValue("AliceVision:tileSize", tileSize)); @@ -372,9 +364,9 @@ int aliceVision_main(int argc, char** argv) std::unique_ptr out_weights = oiio::ImageOutput::create(weightFilepath); // Define output properties - oiio::ImageSpec spec_view(snappedGlobalBbox.width, snappedGlobalBbox.height, 3, (storageDataType == image::EStorageDataType::Half)?oiio::TypeDesc::HALF:oiio::TypeDesc::FLOAT); - oiio::ImageSpec spec_mask(snappedGlobalBbox.width, snappedGlobalBbox.height, 1, oiio::TypeDesc::UCHAR); - oiio::ImageSpec spec_weights(snappedGlobalBbox.width, snappedGlobalBbox.height, 1, oiio::TypeDesc::HALF); + oiio::ImageSpec spec_view(globalBbox.width, globalBbox.height, 3, (storageDataType == image::EStorageDataType::Half)?oiio::TypeDesc::HALF:oiio::TypeDesc::FLOAT); + oiio::ImageSpec spec_mask(globalBbox.width, globalBbox.height, 1, oiio::TypeDesc::UCHAR); + oiio::ImageSpec spec_weights(globalBbox.width, globalBbox.height, 1, oiio::TypeDesc::HALF); spec_view.tile_width = tileSize; spec_view.tile_height = tileSize; @@ -400,13 +392,13 @@ int aliceVision_main(int argc, char** argv) } boxes.clear(); - for (int y = 0; y < snappedGlobalBbox.height; y += tileSize) + for (int y = 0; y < globalBbox.height; y += tileSize) { - for (int x = 0; x < snappedGlobalBbox.width; x += tileSize) + for (int x = 0; x < globalBbox.width; x += tileSize) { BoundingBox localBbox; - localBbox.left = x + snappedGlobalBbox.left; - localBbox.top = y + snappedGlobalBbox.top; + localBbox.left = x + globalBbox.left; + localBbox.top = y + globalBbox.top; localBbox.width = tileSize; localBbox.height = tileSize; boxes.push_back(localBbox); @@ -420,8 +412,8 @@ int aliceVision_main(int argc, char** argv) { BoundingBox localBbox = boxes[boxId]; - int x = localBbox.left - snappedGlobalBbox.left; - int y = localBbox.top - snappedGlobalBbox.top; + int x = localBbox.left - globalBbox.left; + int y = localBbox.top - globalBbox.top; // Prepare coordinates map CoordinatesMap map; From 39f95002865b7513471a84c2617a47dd6dfa1752 Mon Sep 17 00:00:00 2001 From: Fabien Servant Date: Wed, 21 Oct 2020 10:25:25 +0200 Subject: [PATCH 21/79] [panorama] wip no pot --- src/aliceVision/panorama/imageOps.hpp | 7 +- .../panorama/laplacianCompositer.hpp | 8 +- src/aliceVision/panorama/laplacianPyramid.cpp | 389 +++++++++++++----- .../pipeline/main_panoramaCompositing.cpp | 22 +- 4 files changed, 304 insertions(+), 122 deletions(-) diff --git a/src/aliceVision/panorama/imageOps.hpp b/src/aliceVision/panorama/imageOps.hpp index 5eafe22777..a42310b728 100644 --- a/src/aliceVision/panorama/imageOps.hpp +++ b/src/aliceVision/panorama/imageOps.hpp @@ -11,15 +11,20 @@ bool downscale(aliceVision::image::Image& outputColor, const aliceVision::ima { for(int i = 0; i < outputColor.Height(); i++) { + int di = i * 2; + for(int j = 0; j < outputColor.Width(); j++) { - outputColor(i, j) = inputColor(i * 2, j * 2); + int dj = j * 2; + + outputColor(i, j) = inputColor(di, dj); } } return true; } + template bool upscale(aliceVision::image::Image& outputColor, const aliceVision::image::Image& inputColor) { diff --git a/src/aliceVision/panorama/laplacianCompositer.hpp b/src/aliceVision/panorama/laplacianCompositer.hpp index 1c714ff624..d8cf891387 100644 --- a/src/aliceVision/panorama/laplacianCompositer.hpp +++ b/src/aliceVision/panorama/laplacianCompositer.hpp @@ -40,7 +40,7 @@ class LaplacianCompositer : public Compositer size_t minsize = std::min(width, height); const float gaussian_filter_size = 5.0f; size_t optimal_scale = size_t(floor(std::log2(double(minsize) / gaussian_filter_size))); - return optimal_scale; + return 6;//optimal_scale; } @@ -50,7 +50,7 @@ class LaplacianCompositer : public Compositer size_t offset_x, size_t offset_y) { size_t optimalScale = getOptimalScale(color.Width(), color.Height()); - std::cout << optimalScale << std::endl; + std::cout << "---" << optimalScale << std::endl; if(optimalScale < _bands) { ALICEVISION_LOG_ERROR("Decreasing scale !"); @@ -59,11 +59,11 @@ class LaplacianCompositer : public Compositer //If the input scale is more important than previously processed, // The pyramid must be deepened accordingly - /*if(optimalScale > _bands) + if(optimalScale > _bands) { _bands = optimalScale; _pyramidPanorama.augment(_cacheManager, _bands); - }*/ + } // Make sure input is compatible with pyramid processing size_t new_offset_x, new_offset_y; diff --git a/src/aliceVision/panorama/laplacianPyramid.cpp b/src/aliceVision/panorama/laplacianPyramid.cpp index 621e347d96..b8a8beac10 100644 --- a/src/aliceVision/panorama/laplacianPyramid.cpp +++ b/src/aliceVision/panorama/laplacianPyramid.cpp @@ -7,6 +7,111 @@ namespace aliceVision { +template +bool downscale(aliceVision::image::Image& outputColor, const aliceVision::image::Image& inputColor, int posLoop) +{ + for(int i = 0; i < outputColor.Height(); i++) + { + int di = i * 2; + + for(int j = 0; j < outputColor.Width(); j++) + { + int dj = j * 2; + if (dj > posLoop) { + dj = dj - 1; + } + + outputColor(i, j) = inputColor(di, dj); + } + } + + return true; +} + +template +bool upscale(aliceVision::image::Image& outputColor, const aliceVision::image::Image& inputColor, int posLoop) +{ + + size_t width = inputColor.Width(); + size_t height = inputColor.Height(); + size_t dwidth = outputColor.Width(); + size_t dheight = outputColor.Height(); + + for(int i = 0; i < height - 1; i++) + { + int di = i * 2; + + for(int j = 0; j < width - 1; j++) + { + int dj = j * 2; + if (dj >= posLoop) + { + dj = dj - 1; + } + + outputColor(di, dj) = T(); + outputColor(di, dj + 1) = T(); + outputColor(di + 1, dj) = T(); + outputColor(di + 1, dj + 1) = inputColor(i, j); + } + } + + for (int i = 0; i < height; i++) + { + int j = width - 1; + int di = i * 2; + int dj = j * 2; + if (dj >= posLoop) + { + dj = dj - 1; + } + + outputColor(di, dj) = T(); + + if (dj < dwidth - 1) + { + outputColor(di, dj + 1) = T(); + } + + if (di < dheight - 1) + { + outputColor(di + 1, dj) = T(); + + if (dj < dwidth - 1) + { + outputColor(di + 1, dj + 1) = inputColor(i, j); + } + } + } + + for (int j = 0; j < width; j++) + { + int i = height - 1; + int di = i * 2; + int dj = j * 2; + + outputColor(di, dj) = T(); + + if (dj < dwidth - 1) + { + outputColor(di, dj + 1) = T(); + } + + if (di < dheight - 1) + { + outputColor(di + 1, dj) = T(); + + if (dj < dwidth - 1) + { + outputColor(di + 1, dj + 1) = inputColor(i, j); + } + } + } + + return true; +} + + LaplacianPyramid::LaplacianPyramid(size_t base_width, size_t base_height, size_t max_levels) : _baseWidth(base_width), _baseHeight(base_height), @@ -15,7 +120,6 @@ _maxLevels(max_levels) } - bool LaplacianPyramid::initialize(image::TileCacheManager::shared_ptr & cacheManager) { size_t width = _baseWidth; @@ -72,7 +176,6 @@ bool LaplacianPyramid::augment(image::TileCacheManager::shared_ptr & cacheManage int oldMaxLevels = _levels.size(); _maxLevels = newMaxLevels; - //Get content of last level of pyramid CachedImage largerColor; if(!largerColor.createImage(cacheManager, _levels[oldMaxLevels - 1].getWidth(), _levels[oldMaxLevels - 1].getHeight())) @@ -85,6 +188,7 @@ bool LaplacianPyramid::augment(image::TileCacheManager::shared_ptr & cacheManage return false; } + //Get weights of last level of pyramid CachedImage largerWeight; if(!largerWeight.createImage(cacheManager, _weights[oldMaxLevels - 1].getWidth(), _weights[oldMaxLevels - 1].getHeight())) { @@ -96,8 +200,8 @@ bool LaplacianPyramid::augment(image::TileCacheManager::shared_ptr & cacheManage return false; } - //Last level was multiplied by the weight. - //Remove this factor + // Last level was multiplied by the weight. + // Remove this factor largerColor.perPixelOperation(largerWeight, [](const image::RGBfColor & c, const float & w) -> image::RGBfColor { @@ -116,14 +220,14 @@ bool LaplacianPyramid::augment(image::TileCacheManager::shared_ptr & cacheManage } ); - //Create a mask + // Create a mask CachedImage largerMask; if(!largerMask.createImage(cacheManager, largerWeight.getWidth(), largerWeight.getHeight())) { return false; } - //Build the mask + // Build the mask largerMask.perPixelOperation(largerWeight, [](const unsigned char & c, const float & w) -> unsigned char { @@ -131,29 +235,34 @@ bool LaplacianPyramid::augment(image::TileCacheManager::shared_ptr & cacheManage { return 0; } - + return 255; } ); + // Put colors in masked areas if (!feathering(largerColor, largerMask)) { return false; } - - //Augment the number of levels + // Augment the number of levels in the pyramid for (int level = oldMaxLevels; level < newMaxLevels; level++) { CachedImage pyramidImage; CachedImage pyramidWeights; - if(!pyramidImage.createImage(cacheManager, _levels[_levels.size() - 1].getWidth() / 2, _levels[_levels.size() - 1].getHeight() / 2)) + int width = _levels[_levels.size() - 1].getWidth(); + int height = _levels[_levels.size() - 1].getHeight(); + int nextWidth = int(ceil(float(width) / 2.0f)); + int nextHeight = int(ceil(float(height) / 2.0f)); + + if(!pyramidImage.createImage(cacheManager, nextWidth, nextHeight)) { return false; } - if(!pyramidWeights.createImage(cacheManager, _weights[_weights.size() - 1].getWidth() / 2, _weights[_weights.size() - 1].getHeight() / 2)) + if(!pyramidWeights.createImage(cacheManager, nextWidth, nextHeight)) { return false; } @@ -168,21 +277,27 @@ bool LaplacianPyramid::augment(image::TileCacheManager::shared_ptr & cacheManage const int processingSize = 512; const size_t borderSize = 5; - CachedImage currentImage = largerColor; CachedImage currentWeights = largerWeight; for (int level = oldMaxLevels - 1; level < _maxLevels - 1; level++) { + int width = currentImage.getWidth(); + int height = currentImage.getHeight(); + int nextWidth = int(ceil(float(width) / 2.0f)); + int nextHeight = int(ceil(float(height) / 2.0f)); + + + // Create buffer for next level CachedImage nextImage; CachedImage nextWeights; - if(!nextImage.createImage(cacheManager, currentImage.getWidth() / 2, currentImage.getHeight() / 2)) + if(!nextImage.createImage(cacheManager, nextWidth, nextHeight)) { return false; } - if(!nextWeights.createImage(cacheManager, currentImage.getWidth() / 2, currentImage.getHeight() / 2)) + if(!nextWeights.createImage(cacheManager, nextWidth, nextHeight)) { return false; } @@ -190,59 +305,87 @@ bool LaplacianPyramid::augment(image::TileCacheManager::shared_ptr & cacheManage nextImage.fill(image::RGBfColor(0.0f)); nextWeights.fill(0.0f); - for (int y = 0; y < nextImage.getHeight(); y += processingSize) + + //Process image rectangle by rectangle + for (int y = 0; y < currentImage.getHeight(); y += processingSize) { - for (int x = 0; x < nextImage.getWidth(); x += processingSize) + for (int x = 0; x < currentImage.getWidth(); x += processingSize) { + //Compute the initial bouding box for this rectangle BoundingBox nextBbox; nextBbox.left = x; nextBbox.top = y; nextBbox.width = processingSize; nextBbox.height = processingSize; + nextBbox.clampLeft(); + nextBbox.clampTop(); nextBbox.clampRight(nextImage.getWidth() - 1); nextBbox.clampBottom(nextImage.getHeight() - 1); + // Dilate the bounding box to account for filtering BoundingBox dilatedNextBbox = nextBbox.dilate(borderSize); - dilatedNextBbox.clampLeft(); dilatedNextBbox.clampTop(); dilatedNextBbox.clampBottom(nextImage.getHeight() - 1); + if (dilatedNextBbox.width % 2) + { + dilatedNextBbox.width++; + } + + //If the box has a negative left border, + //it is equivalent (with the loop, to shifting at the end) + int leftBorder = nextBbox.left - dilatedNextBbox.left; + int topBorder = nextBbox.top - dilatedNextBbox.top; + if (dilatedNextBbox.left < 0) + { + dilatedNextBbox.left = nextImage.getWidth() + dilatedNextBbox.left; + } + //Same box in the current level BoundingBox currentBbox = nextBbox.doubleSize(); - BoundingBox dilatedCurrentBbox = dilatedNextBbox.doubleSize(); + currentBbox.clampLeft(); + currentBbox.clampTop(); + currentBbox.clampRight(currentImage.getWidth() - 1); + currentBbox.clampBottom(currentImage.getHeight() - 1); + + //Dilated box in the current level + BoundingBox dilatedCurrentBbox = dilatedNextBbox.doubleSize(); + currentBbox.clampTop(); + currentBbox.clampBottom(nextImage.getHeight() - 1); + //Extract the image with borders in the current level aliceVision::image::Image extractedColor(dilatedCurrentBbox.width, dilatedCurrentBbox.height); if (!loopyCachedImageExtract(extractedColor, currentImage, dilatedCurrentBbox)) { return false; } + //Extract the weights with borders in the current level aliceVision::image::Image extractedWeight(dilatedCurrentBbox.width, dilatedCurrentBbox.height); - if (!loopyCachedImageExtract(extractedWeight, currentWeights, dilatedCurrentBbox)) + if (!loopyCachedImageExtract(extractedWeight, currentWeights, currentBbox)) { return false; } - - /*Compute raw next scale from current scale */ + + //Filter current image aliceVision::image::Image buf(dilatedCurrentBbox.width, dilatedCurrentBbox.height); aliceVision::image::Image bufw(dilatedCurrentBbox.width, dilatedCurrentBbox.height); - aliceVision::image::Image colorDownscaled(dilatedNextBbox.width, dilatedNextBbox.height); - aliceVision::image::Image weightDownscaled(dilatedNextBbox.width, dilatedNextBbox.height); - convolveGaussian5x5(buf, extractedColor); convolveGaussian5x5(bufw, extractedWeight); + //Downscale current image to next level + aliceVision::image::Image colorDownscaled(dilatedNextBbox.width, dilatedNextBbox.height); + aliceVision::image::Image weightDownscaled(dilatedNextBbox.width, dilatedNextBbox.height); downscale(colorDownscaled, buf); downscale(weightDownscaled, bufw); + BoundingBox saveBoundingBox; - saveBoundingBox.left = nextBbox.left - dilatedNextBbox.left; - saveBoundingBox.top = nextBbox.top - dilatedNextBbox.top; + saveBoundingBox.left = leftBorder; + saveBoundingBox.top = topBorder; saveBoundingBox.width = nextBbox.width; saveBoundingBox.height = nextBbox.height; - - if (!loopyCachedImageAssign(nextImage, colorDownscaled, nextBbox, saveBoundingBox)) { return false; } @@ -277,8 +420,8 @@ bool LaplacianPyramid::augment(image::TileCacheManager::shared_ptr & cacheManage } - saveBoundingBox.left = currentBbox.left - dilatedCurrentBbox.left; - saveBoundingBox.top = currentBbox.top - dilatedCurrentBbox.top; + saveBoundingBox.left = leftBorder * 2; + saveBoundingBox.top = topBorder * 2; saveBoundingBox.width = currentBbox.width; saveBoundingBox.height = currentBbox.height; @@ -319,14 +462,14 @@ bool LaplacianPyramid::augment(image::TileCacheManager::shared_ptr & cacheManage } -static int pos = 0; + bool LaplacianPyramid::apply(const aliceVision::image::Image& source, const aliceVision::image::Image& mask, - const aliceVision::image::Image& weights, size_t offset_x, size_t offset_y) + const aliceVision::image::Image& weights, + size_t offsetX, size_t offsetY) { //We assume the input source has been feathered //and resized to be a simili power of 2 for the needed scales. - int width = source.Width(); int height = source.Height(); @@ -347,80 +490,114 @@ bool LaplacianPyramid::apply(const aliceVision::image::Image& } } - image::Image current_color = source; - image::Image next_color; - image::Image current_weights = weights; - image::Image next_weights; - image::Image current_mask = mask_float; + image::Image currentColor = source; + image::Image nextColor; + image::Image currentWeights = weights; + image::Image nextWeights; + image::Image currentMask = mask_float; image::Image next_mask; for(int l = 0; l < _levels.size() - 1; l++) { - aliceVision::image::Image buf_masked(width, height); - aliceVision::image::Image buf(width, height); - aliceVision::image::Image buf2(width, height); - aliceVision::image::Image buf_float(width, height); + CachedImage & img = _levels[l]; + + BoundingBox inputBbox; + inputBbox.left = offsetX; + inputBbox.top = offsetY; + inputBbox.width = width; + inputBbox.height = height; + + //If the destination image size is odd, + //It is a big problem + bool specialLoop = false; + int loopPosition = std::numeric_limits::max(); + if (img.getWidth() % 2) + { + //Do we cross the loop ? + if (inputBbox.getRight() > img.getWidth()) + { + specialLoop = true; + loopPosition = img.getWidth() - inputBbox.left; + } + } - next_color = aliceVision::image::Image(width / 2, height / 2); - next_weights = aliceVision::image::Image(width / 2, height / 2); - next_mask = aliceVision::image::Image(width / 2, height / 2); + image::Image bufMasked(width, height); + image::Image buf(width, height); + image::Image buf2(width, height); + image::Image bufFloat(width, height); + /*Apply mask to content before convolution*/ - for(int i = 0; i < current_color.Height(); i++) + for(int i = 0; i < currentColor.Height(); i++) { - for(int j = 0; j < current_color.Width(); j++) + for(int j = 0; j < currentColor.Width(); j++) { - if(std::abs(current_mask(i, j)) > 1e-6) + if(std::abs(currentMask(i, j)) > 1e-6) { - buf_masked(i, j) = current_color(i, j); + bufMasked(i, j) = currentColor(i, j); } else { - buf_masked(i, j).r() = 0.0f; - buf_masked(i, j).g() = 0.0f; - buf_masked(i, j).b() = 0.0f; - current_weights(i, j) = 0.0f; + bufMasked(i, j).r() = 0.0f; + bufMasked(i, j).g() = 0.0f; + bufMasked(i, j).b() = 0.0f; + currentWeights(i, j) = 0.0f; } } } - convolveGaussian5x5(buf, buf_masked); - convolveGaussian5x5(buf_float, current_mask); + convolveGaussian5x5(buf, bufMasked); + convolveGaussian5x5(bufFloat, currentMask); - /* - Normalize given mask - */ - for(int i = 0; i < current_color.Height(); i++) + //Normalize given mask + //(Make sure the convolution sum is 1) + for(int i = 0; i < currentColor.Height(); i++) { - for(int j = 0; j < current_color.Width(); j++) + for(int j = 0; j < currentColor.Width(); j++) { - - float m = buf_float(i, j); + float m = bufFloat(i, j); if(std::abs(m) > 1e-6) { buf(i, j).r() = buf(i, j).r() / m; buf(i, j).g() = buf(i, j).g() / m; buf(i, j).b() = buf(i, j).b() / m; - buf_float(i, j) = 1.0f; + bufFloat(i, j) = 1.0f; } else { buf(i, j).r() = 0.0f; buf(i, j).g() = 0.0f; buf(i, j).b() = 0.0f; - buf_float(i, j) = 0.0f; + bufFloat(i, j) = 0.0f; } } } - downscale(next_color, buf); - downscale(next_mask, buf_float); - upscale(buf, next_color); + int offset = 0; + if (specialLoop) + { + offset = 1; + } + + int nextWidth = int(floor(float(width + offset) / 2.0f)); + int nextHeight = int(floor(float(height) / 2.0f)); + + nextColor = aliceVision::image::Image(nextWidth, nextHeight); + nextWeights = aliceVision::image::Image(nextWidth, nextHeight); + next_mask = aliceVision::image::Image(nextWidth, nextHeight); + + downscale(nextColor, buf, loopPosition); + downscale(next_mask, bufFloat, loopPosition); + upscale(buf, nextColor, loopPosition); + + //Filter convolveGaussian5x5(buf2, buf); + //Values must be multiplied by 4 as our upscale was using + //filling of 0 values for(int i = 0; i < buf2.Height(); i++) { for(int j = 0; j < buf2.Width(); j++) @@ -429,39 +606,43 @@ bool LaplacianPyramid::apply(const aliceVision::image::Image& } } - substract(current_color, current_color, buf2); + //Only keep the difference (Band pass) + substract(currentColor, currentColor, buf2); + //Downscale weights + convolveGaussian5x5(bufFloat, currentWeights); + downscale(nextWeights, bufFloat, loopPosition); - convolveGaussian5x5(buf_float, current_weights); - downscale(next_weights, buf_float); - - if (!merge(current_color, current_weights, l, offset_x, offset_y)) + //Merge this view with previous ones + if (!merge(currentColor, currentWeights, l, offsetX, offsetY)) { return false; } - current_color = next_color; - current_weights = next_weights; - current_mask = next_mask; - - width /= 2; - height /= 2; - offset_x /= 2; - offset_y /= 2; + //Swap buffers + currentColor = nextColor; + currentWeights = nextWeights; + currentMask = next_mask; + width = nextWidth; + height = nextHeight; + + //Thanks to previous operation, + //We are sure the offset is a power of two for the required levels + offsetX = offsetX / 2; + offsetY = offsetY / 2; } - if (!merge(current_color, current_weights, _levels.size() - 1, offset_x, offset_y)) + if (!merge(currentColor, currentWeights, _levels.size() - 1, offsetX, offsetY)) { return false; } - pos++; return true; } bool LaplacianPyramid::merge(const aliceVision::image::Image& oimg, - const aliceVision::image::Image& oweight, size_t level, size_t offset_x, - size_t offset_y) + const aliceVision::image::Image& oweight, size_t level, size_t offsetX, + size_t offsetY) { CachedImage & img = _levels[level]; CachedImage & weight = _weights[level]; @@ -470,8 +651,8 @@ bool LaplacianPyramid::merge(const aliceVision::image::Image& aliceVision::image::Image extractedWeight(oimg.Width(), oimg.Height()); BoundingBox extractBb; - extractBb.left = offset_x; - extractBb.top = offset_y; + extractBb.left = offsetX; + extractBb.top = offsetY; extractBb.width = oimg.Width(); extractBb.height = oimg.Height(); extractBb.clampBottom(img.getHeight() - 1); @@ -491,34 +672,14 @@ bool LaplacianPyramid::merge(const aliceVision::image::Image& return false; } - bool special = false; - if (img.getWidth() % 2) - { - special = true; - } - for(int y = 0; y < inputBb.height; y++) { - int vx = 0; - for(int x = 0; x < inputBb.width && vx < inputBb.width; x++) - { - //Special process for levels with a width non divided by 2 - //To account for the upscale difference, we "add" a black column - if (special) - { - int posX = offset_x + x; - if (posX == img.getWidth() - 1) - { - vx++; - } - } - - extractedColor(y, x).r() += oimg(y, vx).r() * oweight(y, vx); - extractedColor(y, x).g() += oimg(y, vx).g() * oweight(y, vx); - extractedColor(y, x).b() += oimg(y, vx).b() * oweight(y, vx); - extractedWeight(y, x) += oweight(y, vx); - - vx++; + for(int x = 0; x < inputBb.width; x++) + { + extractedColor(y, x).r() += oimg(y, x).r() * oweight(y, x); + extractedColor(y, x).g() += oimg(y, x).g() * oweight(y, x); + extractedColor(y, x).b() += oimg(y, x).b() * oweight(y, x); + extractedWeight(y, x) += oweight(y, x); } } @@ -557,12 +718,12 @@ bool LaplacianPyramid::rebuild(CachedImage& output) ); } - - removeNegativeValues(_levels[_levels.size() - 1]); for(int l = _levels.size() - 2; l >= 0; l--) { + + std::cout << _levels[l].getWidth() << " " << _levels[l].getHeight() << std::endl; const size_t processingSize = 512; const size_t borderSize = 5; diff --git a/src/software/pipeline/main_panoramaCompositing.cpp b/src/software/pipeline/main_panoramaCompositing.cpp index 5cb2eab040..74694314e7 100644 --- a/src/software/pipeline/main_panoramaCompositing.cpp +++ b/src/software/pipeline/main_panoramaCompositing.cpp @@ -316,7 +316,7 @@ int aliceVision_main(int argc, char** argv) // Configure the cache manager memory cacheManager->setInCoreMaxObjectCount(1000); - LaplacianCompositer compositer(cacheManager, panoramaSize.first, panoramaSize.second, 5); + LaplacianCompositer compositer(cacheManager, panoramaSize.first, panoramaSize.second, 6); if (!compositer.initialize()) { @@ -352,13 +352,29 @@ int aliceVision_main(int argc, char** argv) std::vector> viewOrderedByScale; { std::map>> mapViewsScale; - for(const auto& it : sfmData.getViews()) + for(const auto & it : sfmData.getViews()) { - size_t scale = compositer.getOptimalScale(it.second->getWidth(), it.second->getHeight()); + auto view = it.second; + IndexT viewId = view->getViewId(); + + if(!sfmData.isPoseAndIntrinsicDefined(view.get())) + { + // skip unreconstructed views + continue; + } + + // Load mask + const std::string maskPath = (fs::path(warpingFolder) / (std::to_string(viewId) + "_mask.exr")).string(); + image::Image mask; + image::readImage(maskPath, mask, image::EImageColorSpace::NO_CONVERSION); + + //Estimate scale + size_t scale = compositer.getOptimalScale(mask.Width(), mask.Height()); mapViewsScale[scale].push_back(it.second); } for (auto scaledList : mapViewsScale) { + std::cout << scaledList.first << std::endl; for (auto item : scaledList.second) { viewOrderedByScale.push_back(item); From d40a88ce9ba489964dcab833beba1591f863999e Mon Sep 17 00:00:00 2001 From: Fabien Servant Date: Thu, 22 Oct 2020 09:55:31 +0200 Subject: [PATCH 22/79] [panorama] working non pot --- src/aliceVision/panorama/feathering.cpp | 10 +- .../panorama/laplacianCompositer.hpp | 3 +- src/aliceVision/panorama/laplacianPyramid.cpp | 313 ++++++++---------- src/aliceVision/panorama/laplacianPyramid.hpp | 1 + .../pipeline/main_panoramaCompositing.cpp | 61 ++-- 5 files changed, 182 insertions(+), 206 deletions(-) diff --git a/src/aliceVision/panorama/feathering.cpp b/src/aliceVision/panorama/feathering.cpp index fba6f98b34..38c21c5045 100644 --- a/src/aliceVision/panorama/feathering.cpp +++ b/src/aliceVision/panorama/feathering.cpp @@ -7,7 +7,6 @@ bool feathering(aliceVision::image::Image& output, const aliceVision::image::Image& color, const aliceVision::image::Image& inputMask) { - std::vector> feathering; std::vector> feathering_mask; feathering.push_back(color); @@ -17,7 +16,7 @@ bool feathering(aliceVision::image::Image& output, int width = color.Width(); int height = color.Height(); - while(1) + while (!(width < 2 || height < 2)) { const image::Image& src = feathering[lvl]; const image::Image& src_mask = feathering_mask[lvl]; @@ -78,9 +77,6 @@ bool feathering(aliceVision::image::Image& output, width = half.Width(); height = half.Height(); - if(width < 2 || height < 2) - break; - lvl++; } @@ -296,11 +292,12 @@ bool feathering(CachedImage & input_output, CachedImage rowColor = tilesColor[i]; @@ -397,6 +394,7 @@ bool feathering(CachedImage & input_output, CachedImage= 0; lvl--) { diff --git a/src/aliceVision/panorama/laplacianCompositer.hpp b/src/aliceVision/panorama/laplacianCompositer.hpp index d8cf891387..1bf9bf703e 100644 --- a/src/aliceVision/panorama/laplacianCompositer.hpp +++ b/src/aliceVision/panorama/laplacianCompositer.hpp @@ -40,7 +40,7 @@ class LaplacianCompositer : public Compositer size_t minsize = std::min(width, height); const float gaussian_filter_size = 5.0f; size_t optimal_scale = size_t(floor(std::log2(double(minsize) / gaussian_filter_size))); - return 6;//optimal_scale; + return optimal_scale; } @@ -50,7 +50,6 @@ class LaplacianCompositer : public Compositer size_t offset_x, size_t offset_y) { size_t optimalScale = getOptimalScale(color.Width(), color.Height()); - std::cout << "---" << optimalScale << std::endl; if(optimalScale < _bands) { ALICEVISION_LOG_ERROR("Decreasing scale !"); diff --git a/src/aliceVision/panorama/laplacianPyramid.cpp b/src/aliceVision/panorama/laplacianPyramid.cpp index b8a8beac10..a44e630acb 100644 --- a/src/aliceVision/panorama/laplacianPyramid.cpp +++ b/src/aliceVision/panorama/laplacianPyramid.cpp @@ -7,111 +7,6 @@ namespace aliceVision { -template -bool downscale(aliceVision::image::Image& outputColor, const aliceVision::image::Image& inputColor, int posLoop) -{ - for(int i = 0; i < outputColor.Height(); i++) - { - int di = i * 2; - - for(int j = 0; j < outputColor.Width(); j++) - { - int dj = j * 2; - if (dj > posLoop) { - dj = dj - 1; - } - - outputColor(i, j) = inputColor(di, dj); - } - } - - return true; -} - -template -bool upscale(aliceVision::image::Image& outputColor, const aliceVision::image::Image& inputColor, int posLoop) -{ - - size_t width = inputColor.Width(); - size_t height = inputColor.Height(); - size_t dwidth = outputColor.Width(); - size_t dheight = outputColor.Height(); - - for(int i = 0; i < height - 1; i++) - { - int di = i * 2; - - for(int j = 0; j < width - 1; j++) - { - int dj = j * 2; - if (dj >= posLoop) - { - dj = dj - 1; - } - - outputColor(di, dj) = T(); - outputColor(di, dj + 1) = T(); - outputColor(di + 1, dj) = T(); - outputColor(di + 1, dj + 1) = inputColor(i, j); - } - } - - for (int i = 0; i < height; i++) - { - int j = width - 1; - int di = i * 2; - int dj = j * 2; - if (dj >= posLoop) - { - dj = dj - 1; - } - - outputColor(di, dj) = T(); - - if (dj < dwidth - 1) - { - outputColor(di, dj + 1) = T(); - } - - if (di < dheight - 1) - { - outputColor(di + 1, dj) = T(); - - if (dj < dwidth - 1) - { - outputColor(di + 1, dj + 1) = inputColor(i, j); - } - } - } - - for (int j = 0; j < width; j++) - { - int i = height - 1; - int di = i * 2; - int dj = j * 2; - - outputColor(di, dj) = T(); - - if (dj < dwidth - 1) - { - outputColor(di, dj + 1) = T(); - } - - if (di < dheight - 1) - { - outputColor(di + 1, dj) = T(); - - if (dj < dwidth - 1) - { - outputColor(di + 1, dj + 1) = inputColor(i, j); - } - } - } - - return true; -} - - LaplacianPyramid::LaplacianPyramid(size_t base_width, size_t base_height, size_t max_levels) : _baseWidth(base_width), _baseHeight(base_height), @@ -134,6 +29,15 @@ bool LaplacianPyramid::initialize(image::TileCacheManager::shared_ptr & cacheMan CachedImage color; CachedImage weights; + //If the level width is odd, it is a problem. + //Let update this level to an even size. we'll manage it later + //Store the real size to know we updated it + _realWidths.push_back(width); + if (width % 2) + { + width++; + } + if(!color.createImage(cacheManager, width, height)) { return false; @@ -157,8 +61,8 @@ bool LaplacianPyramid::initialize(image::TileCacheManager::shared_ptr & cacheMan _levels.push_back(color); _weights.push_back(weights); + width = width / 2; height = int(ceil(float(height) / 2.0f)); - width = int(ceil(float(width) / 2.0f)); } return true; @@ -240,6 +144,7 @@ bool LaplacianPyramid::augment(image::TileCacheManager::shared_ptr & cacheManage } ); + // Put colors in masked areas if (!feathering(largerColor, largerMask)) { @@ -254,9 +159,19 @@ bool LaplacianPyramid::augment(image::TileCacheManager::shared_ptr & cacheManage int width = _levels[_levels.size() - 1].getWidth(); int height = _levels[_levels.size() - 1].getHeight(); - int nextWidth = int(ceil(float(width) / 2.0f)); + + int nextWidth = width / 2; int nextHeight = int(ceil(float(height) / 2.0f)); + //If the level width is odd, it is a problem. + //Let update this level to an even size. we'll manage it later + //Store the real size to know we updated it + _realWidths.push_back(nextWidth); + if (nextWidth % 2) + { + nextWidth++; + } + if(!pyramidImage.createImage(cacheManager, nextWidth, nextHeight)) { return false; @@ -274,6 +189,7 @@ bool LaplacianPyramid::augment(image::TileCacheManager::shared_ptr & cacheManage _weights.push_back(pyramidWeights); } + std::cout << "ok" << std::endl; const int processingSize = 512; const size_t borderSize = 5; @@ -307,9 +223,9 @@ bool LaplacianPyramid::augment(image::TileCacheManager::shared_ptr & cacheManage //Process image rectangle by rectangle - for (int y = 0; y < currentImage.getHeight(); y += processingSize) + for (int y = 0; y < nextImage.getHeight(); y += processingSize) { - for (int x = 0; x < currentImage.getWidth(); x += processingSize) + for (int x = 0; x < nextImage.getWidth(); x += processingSize) { //Compute the initial bouding box for this rectangle BoundingBox nextBbox; @@ -322,14 +238,11 @@ bool LaplacianPyramid::augment(image::TileCacheManager::shared_ptr & cacheManage nextBbox.clampRight(nextImage.getWidth() - 1); nextBbox.clampBottom(nextImage.getHeight() - 1); + // Dilate the bounding box to account for filtering BoundingBox dilatedNextBbox = nextBbox.dilate(borderSize); dilatedNextBbox.clampTop(); dilatedNextBbox.clampBottom(nextImage.getHeight() - 1); - if (dilatedNextBbox.width % 2) - { - dilatedNextBbox.width++; - } //If the box has a negative left border, //it is equivalent (with the loop, to shifting at the end) @@ -337,7 +250,7 @@ bool LaplacianPyramid::augment(image::TileCacheManager::shared_ptr & cacheManage int topBorder = nextBbox.top - dilatedNextBbox.top; if (dilatedNextBbox.left < 0) { - dilatedNextBbox.left = nextImage.getWidth() + dilatedNextBbox.left; + dilatedNextBbox.left += nextImage.getWidth(); } //Same box in the current level @@ -350,8 +263,8 @@ bool LaplacianPyramid::augment(image::TileCacheManager::shared_ptr & cacheManage //Dilated box in the current level BoundingBox dilatedCurrentBbox = dilatedNextBbox.doubleSize(); currentBbox.clampTop(); - currentBbox.clampBottom(nextImage.getHeight() - 1); - + currentBbox.clampBottom(currentImage.getHeight() - 1); + //Extract the image with borders in the current level aliceVision::image::Image extractedColor(dilatedCurrentBbox.width, dilatedCurrentBbox.height); if (!loopyCachedImageExtract(extractedColor, currentImage, dilatedCurrentBbox)) @@ -361,12 +274,11 @@ bool LaplacianPyramid::augment(image::TileCacheManager::shared_ptr & cacheManage //Extract the weights with borders in the current level aliceVision::image::Image extractedWeight(dilatedCurrentBbox.width, dilatedCurrentBbox.height); - if (!loopyCachedImageExtract(extractedWeight, currentWeights, currentBbox)) + if (!loopyCachedImageExtract(extractedWeight, currentWeights, dilatedCurrentBbox)) { return false; } - //Filter current image aliceVision::image::Image buf(dilatedCurrentBbox.width, dilatedCurrentBbox.height); aliceVision::image::Image bufw(dilatedCurrentBbox.width, dilatedCurrentBbox.height); @@ -379,7 +291,6 @@ bool LaplacianPyramid::augment(image::TileCacheManager::shared_ptr & cacheManage downscale(colorDownscaled, buf); downscale(weightDownscaled, bufw); - BoundingBox saveBoundingBox; saveBoundingBox.left = leftBorder; saveBoundingBox.top = topBorder; @@ -403,6 +314,7 @@ bool LaplacianPyramid::augment(image::TileCacheManager::shared_ptr & cacheManage for (int i = 0; i < buf2.Height(); i++) { for (int j = 0; j < buf2.Width(); j++) { + buf2(i,j) *= 4.0f; } } @@ -419,7 +331,6 @@ bool LaplacianPyramid::augment(image::TileCacheManager::shared_ptr & cacheManage } } - saveBoundingBox.left = leftBorder * 2; saveBoundingBox.top = topBorder * 2; saveBoundingBox.width = currentBbox.width; @@ -490,13 +401,13 @@ bool LaplacianPyramid::apply(const aliceVision::image::Image& } } + image::Image currentColor = source; image::Image nextColor; image::Image currentWeights = weights; image::Image nextWeights; image::Image currentMask = mask_float; - image::Image next_mask; - + image::Image nextMask; for(int l = 0; l < _levels.size() - 1; l++) { @@ -512,26 +423,70 @@ bool LaplacianPyramid::apply(const aliceVision::image::Image& //It is a big problem bool specialLoop = false; int loopPosition = std::numeric_limits::max(); - if (img.getWidth() % 2) + int realLevelWidth = _realWidths[l]; + int offset = 0; + + if (img.getWidth() != realLevelWidth) { //Do we cross the loop ? if (inputBbox.getRight() > img.getWidth()) { specialLoop = true; - loopPosition = img.getWidth() - inputBbox.left; + loopPosition = realLevelWidth - inputBbox.left; + offset = 2; } } - image::Image bufMasked(width, height); - image::Image buf(width, height); - image::Image buf2(width, height); - image::Image bufFloat(width, height); + //Create aligned images if necessary + if (specialLoop) + { + image::Image alignedColor(width + offset, height); + image::Image alignedWeights(width + offset, height); + image::Image alignedMask(width + offset, height); + + for(int i = 0; i < currentColor.Height(); i++) + { + int dj = 0; + + for(int j = 0; j < currentColor.Width(); j++) + { + if (j == loopPosition) + { + dj++; + } + + alignedMask(i, dj) = currentMask(i, j); + alignedColor(i, dj) = currentColor(i, j); + alignedWeights(i, dj) = currentWeights(i, j); + + dj++; + } + + if (specialLoop) + { + alignedColor(i, loopPosition) = alignedColor(i, loopPosition + 1); + alignedMask(i, loopPosition) = alignedMask(i, loopPosition + 1); + alignedWeights(i, loopPosition) = alignedWeights(i, loopPosition + 1); + alignedMask(i, width + 1) = 0; + alignedWeights(i, width + 1) = 0; + } + } + + currentColor = alignedColor; + currentWeights = alignedWeights; + currentMask = alignedMask; + } + + + image::Image bufMasked(width + offset, height); + image::Image buf(width + offset, height); + image::Image buf2(width + offset, height); + image::Image bufFloat(width + offset, height); - /*Apply mask to content before convolution*/ - for(int i = 0; i < currentColor.Height(); i++) + for(int i = 0; i < height; i++) { - for(int j = 0; j < currentColor.Width(); j++) + for(int j = 0; j < width; j++) { if(std::abs(currentMask(i, j)) > 1e-6) { @@ -546,15 +501,23 @@ bool LaplacianPyramid::apply(const aliceVision::image::Image& } } } + + + if (!convolveGaussian5x5(buf, bufMasked)) + { + return false; + } - convolveGaussian5x5(buf, bufMasked); - convolveGaussian5x5(bufFloat, currentMask); + if (!convolveGaussian5x5(bufFloat, currentMask)) + { + return false; + } //Normalize given mask //(Make sure the convolution sum is 1) - for(int i = 0; i < currentColor.Height(); i++) + for(int i = 0; i < buf.Height(); i++) { - for(int j = 0; j < currentColor.Width(); j++) + for(int j = 0; j < buf.Width(); j++) { float m = bufFloat(i, j); @@ -576,42 +539,60 @@ bool LaplacianPyramid::apply(const aliceVision::image::Image& } - int offset = 0; - if (specialLoop) - { - offset = 1; - } - - int nextWidth = int(floor(float(width + offset) / 2.0f)); + int nextWidth = width / 2; int nextHeight = int(floor(float(height) / 2.0f)); nextColor = aliceVision::image::Image(nextWidth, nextHeight); nextWeights = aliceVision::image::Image(nextWidth, nextHeight); - next_mask = aliceVision::image::Image(nextWidth, nextHeight); + nextMask = aliceVision::image::Image(nextWidth, nextHeight); + + if (!downscale(nextColor, buf)) + { + return false; + } + + if (!downscale(nextMask, bufFloat)) + { + return false; + } - downscale(nextColor, buf, loopPosition); - downscale(next_mask, bufFloat, loopPosition); - upscale(buf, nextColor, loopPosition); + if (!upscale(buf, nextColor)) + { + return false; + } //Filter - convolveGaussian5x5(buf2, buf); + if (!convolveGaussian5x5(buf2, buf)) + { + return false; + } //Values must be multiplied by 4 as our upscale was using //filling of 0 values for(int i = 0; i < buf2.Height(); i++) { for(int j = 0; j < buf2.Width(); j++) - { + { buf2(i, j) *= 4.0f; } } //Only keep the difference (Band pass) - substract(currentColor, currentColor, buf2); + if (!substract(currentColor, currentColor, buf2)) + { + return false; + } //Downscale weights - convolveGaussian5x5(bufFloat, currentWeights); - downscale(nextWeights, bufFloat, loopPosition); + if (!convolveGaussian5x5(bufFloat, currentWeights)) + { + return false; + } + + if (!downscale(nextWeights, bufFloat)) + { + return false; + } //Merge this view with previous ones if (!merge(currentColor, currentWeights, l, offsetX, offsetY)) @@ -622,7 +603,7 @@ bool LaplacianPyramid::apply(const aliceVision::image::Image& //Swap buffers currentColor = nextColor; currentWeights = nextWeights; - currentMask = next_mask; + currentMask = nextMask; width = nextWidth; height = nextHeight; @@ -722,23 +703,12 @@ bool LaplacianPyramid::rebuild(CachedImage& output) for(int l = _levels.size() - 2; l >= 0; l--) { - - std::cout << _levels[l].getWidth() << " " << _levels[l].getHeight() << std::endl; const size_t processingSize = 512; const size_t borderSize = 5; int halfLevel = l + 1; int currentLevel = l; - int x = 0; - int y = 0; - - bool specialBorder = false; - if (_levels[halfLevel].getWidth() % 2) - { - specialBorder = true; - } - for (int py = 0; py < _levels[halfLevel].getHeight(); py += processingSize) { for (int px = 0; px < _levels[halfLevel].getWidth(); px += processingSize) @@ -790,8 +760,16 @@ bool LaplacianPyramid::rebuild(CachedImage& output) aliceVision::image::Image buf(dilatedBb.width * 2, dilatedBb.height * 2); aliceVision::image::Image buf2(dilatedBb.width * 2, dilatedBb.height * 2); - upscale(buf, extracted); - convolveGaussian5x5(buf2, buf, false); + if (!upscale(buf, extracted)) + { + return false; + } + + + if (!convolveGaussian5x5(buf2, buf, false)) + { + return false; + } for(int y = 0; y < buf2.Height(); y++) { @@ -823,13 +801,8 @@ bool LaplacianPyramid::rebuild(CachedImage& output) } removeNegativeValues(_levels[currentLevel]); - - } - - - for(int i = 0; i < output.getTiles().size(); i++) { diff --git a/src/aliceVision/panorama/laplacianPyramid.hpp b/src/aliceVision/panorama/laplacianPyramid.hpp index 0faa5ae399..46b7ffcc5e 100644 --- a/src/aliceVision/panorama/laplacianPyramid.hpp +++ b/src/aliceVision/panorama/laplacianPyramid.hpp @@ -32,6 +32,7 @@ class LaplacianPyramid std::vector> _levels; std::vector> _weights; + std::vector _realWidths; }; } // namespace aliceVision \ No newline at end of file diff --git a/src/software/pipeline/main_panoramaCompositing.cpp b/src/software/pipeline/main_panoramaCompositing.cpp index 74694314e7..a0b94985f7 100644 --- a/src/software/pipeline/main_panoramaCompositing.cpp +++ b/src/software/pipeline/main_panoramaCompositing.cpp @@ -124,18 +124,13 @@ bool computeWTALabels(CachedImage & labels, image::TileCacheManager::sha return true; } -bool computeGCLabels(CachedImage & labels, image::TileCacheManager::shared_ptr& cacheManager, const sfmData::SfMData& sfmData, const std::string & inputPath, std::pair & panoramaSize) +bool computeGCLabels(CachedImage & labels, image::TileCacheManager::shared_ptr& cacheManager, const sfmData::SfMData& sfmData, const std::string & inputPath, std::pair & panoramaSize, int smallestViewScale) { - //Compute coarsest level possible for graph cut - int initial_level = 0; - int min_width_for_graphcut = 1000; - double ratio = double(panoramaSize.first) / double(min_width_for_graphcut); - if (ratio > 1.0) { - initial_level = int(floor(log2(ratio))); - } + int pyramidSize = std::max(0, smallestViewScale - 1); + ALICEVISION_LOG_INFO("Graphcut pyramid size is " << pyramidSize); - HierarchicalGraphcutSeams seams(cacheManager, panoramaSize.first, panoramaSize.second, initial_level + 1); + HierarchicalGraphcutSeams seams(cacheManager, panoramaSize.first, panoramaSize.second, pyramidSize); if (!seams.initialize()) { @@ -316,7 +311,7 @@ int aliceVision_main(int argc, char** argv) // Configure the cache manager memory cacheManager->setInCoreMaxObjectCount(1000); - LaplacianCompositer compositer(cacheManager, panoramaSize.first, panoramaSize.second, 6); + LaplacianCompositer compositer(cacheManager, panoramaSize.first, panoramaSize.second, 5); if (!compositer.initialize()) { @@ -331,24 +326,8 @@ int aliceVision_main(int argc, char** argv) return EXIT_FAILURE; } - CachedImage labels; - if (!computeWTALabels(labels, cacheManager, sfmData, warpingFolder, panoramaSize)) - { - ALICEVISION_LOG_ERROR("Error computing initial labels"); - return EXIT_FAILURE; - } - - labels.writeImage("/home/mmoc/labels_wta.exr"); - - /*if (!computeGCLabels(labels, cacheManager, sfmData, warpingFolder, panoramaSize)) - { - ALICEVISION_LOG_ERROR("Error computing graph cut labels"); - return EXIT_FAILURE; - } - - labels.writeImage("/home/mmoc/labels_gc.exr");*/ - //Get a list of views ordered by their image scale + size_t smallestScale = 1; std::vector> viewOrderedByScale; { std::map>> mapViewsScale; @@ -372,9 +351,17 @@ int aliceVision_main(int argc, char** argv) size_t scale = compositer.getOptimalScale(mask.Width(), mask.Height()); mapViewsScale[scale].push_back(it.second); } + + if (mapViewsScale.size() == 0) + { + ALICEVISION_LOG_ERROR("No valid view"); + return EXIT_FAILURE; + } + + smallestScale = mapViewsScale.begin()->first; + for (auto scaledList : mapViewsScale) { - std::cout << scaledList.first << std::endl; for (auto item : scaledList.second) { viewOrderedByScale.push_back(item); @@ -382,6 +369,24 @@ int aliceVision_main(int argc, char** argv) } } + CachedImage labels; + if (!computeWTALabels(labels, cacheManager, sfmData, warpingFolder, panoramaSize)) + { + ALICEVISION_LOG_ERROR("Error computing initial labels"); + return EXIT_FAILURE; + } + + labels.writeImage("/home/mmoc/labels_wta.exr"); + + + if (!computeGCLabels(labels, cacheManager, sfmData, warpingFolder, panoramaSize, smallestScale)) + { + ALICEVISION_LOG_ERROR("Error computing graph cut labels"); + return EXIT_FAILURE; + } + + labels.writeImage("/home/mmoc/labels_gc.exr"); + size_t pos = 0; std::cout << "compositing" << std::endl; From 007bd21f7f4fd9c1d0c00941029091d0d73ea3e7 Mon Sep 17 00:00:00 2001 From: Fabien Servant Date: Thu, 22 Oct 2020 18:12:10 +0200 Subject: [PATCH 23/79] [panorama] add options for compositing --- src/aliceVision/panorama/compositer.hpp | 18 +- src/aliceVision/panorama/graphcut.hpp | 83 +----- .../panorama/laplacianCompositer.hpp | 5 +- src/aliceVision/panorama/laplacianPyramid.cpp | 2 - src/aliceVision/panorama/seams.cpp | 270 +++++++++++++++--- src/aliceVision/panorama/seams.hpp | 5 +- .../pipeline/main_panoramaCompositing.cpp | 202 ++++++++----- 7 files changed, 378 insertions(+), 207 deletions(-) diff --git a/src/aliceVision/panorama/compositer.hpp b/src/aliceVision/panorama/compositer.hpp index 7c5e0212ba..16b6e9a3c9 100644 --- a/src/aliceVision/panorama/compositer.hpp +++ b/src/aliceVision/panorama/compositer.hpp @@ -4,6 +4,7 @@ #include "cachedImage.hpp" #include "imageOps.hpp" +#include "seams.hpp" namespace aliceVision { @@ -25,7 +26,7 @@ class Compositer int offset_x, int offset_y) { aliceVision::image::Image masked(color.Width(), color.Height()); - + BoundingBox panoramaBb; panoramaBb.left = offset_x; panoramaBb.top = offset_y; @@ -99,6 +100,21 @@ class Compositer return true; } + virtual size_t getOptimalScale(int width, int height) + { + return 1; + } + + bool drawBorders(const aliceVision::image::Image& mask, size_t offsetX, size_t offsetY) + { + return ::aliceVision::drawBorders(_panorama, mask, offsetX, offsetY); + } + + bool drawSeams(CachedImage& label) + { + return ::aliceVision::drawSeams(_panorama, label); + } + protected: image::TileCacheManager::shared_ptr _cacheManager; CachedImage _panorama; diff --git a/src/aliceVision/panorama/graphcut.hpp b/src/aliceVision/panorama/graphcut.hpp index a9e3a1aa04..cb00803496 100644 --- a/src/aliceVision/panorama/graphcut.hpp +++ b/src/aliceVision/panorama/graphcut.hpp @@ -8,6 +8,7 @@ #include "distance.hpp" #include "boundingBox.hpp" #include "imageOps.hpp" +#include "seams.hpp" namespace aliceVision { @@ -131,85 +132,7 @@ class MaxFlow_AdjList const NodeType _T; //< fullness }; -bool computeSeamsMap(image::Image& seams, const image::Image& labels) -{ - - if(seams.size() != labels.size()) - { - return false; - } - - seams.fill(0); - - for(int j = 1; j < labels.Width() - 1; j++) - { - IndexT label = labels(0, j); - IndexT same = true; - - same &= (labels(0, j - 1) == label); - same &= (labels(0, j + 1) == label); - same &= (labels(1, j - 1) == label); - same &= (labels(1, j) == label); - same &= (labels(1, j + 1) == label); - - if(same) - { - continue; - } - - seams(0, j) = 255; - } - - int lastrow = labels.Height() - 1; - for(int j = 1; j < labels.Width() - 1; j++) - { - IndexT label = labels(lastrow, j); - IndexT same = true; - - same &= (labels(lastrow - 1, j - 1) == label); - same &= (labels(lastrow - 1, j + 1) == label); - same &= (labels(lastrow, j - 1) == label); - same &= (labels(lastrow, j) == label); - same &= (labels(lastrow, j + 1) == label); - - if(same) - { - continue; - } - - seams(lastrow, j) = 255; - } - - for(int i = 1; i < labels.Height() - 1; i++) - { - - for(int j = 1; j < labels.Width() - 1; j++) - { - - IndexT label = labels(i, j); - IndexT same = true; - - same &= (labels(i - 1, j - 1) == label); - same &= (labels(i - 1, j) == label); - same &= (labels(i - 1, j + 1) == label); - same &= (labels(i, j - 1) == label); - same &= (labels(i, j + 1) == label); - same &= (labels(i + 1, j - 1) == label); - same &= (labels(i + 1, j) == label); - same &= (labels(i + 1, j + 1) == label); - - if(same) - { - continue; - } - - seams(i, j) = 255; - } - } - - return true; -} - +bool computeSeamsMap(image::Image& seams, const image::Image& labels); class GraphcutSeams { @@ -528,7 +451,7 @@ class GraphcutSeams } } } - + double oldCost = cost(localLabels, graphCutInput, input.id); if (!alphaExpansion(localLabels, distanceMap, graphCutInput, input.id)) { diff --git a/src/aliceVision/panorama/laplacianCompositer.hpp b/src/aliceVision/panorama/laplacianCompositer.hpp index 1bf9bf703e..ea728eb049 100644 --- a/src/aliceVision/panorama/laplacianCompositer.hpp +++ b/src/aliceVision/panorama/laplacianCompositer.hpp @@ -27,7 +27,7 @@ class LaplacianCompositer : public Compositer return _pyramidPanorama.initialize(_cacheManager); } - size_t getOptimalScale(int width, int height) + virtual size_t getOptimalScale(int width, int height) { /* Look for the smallest scale such that the image is not smaller than the @@ -47,7 +47,7 @@ class LaplacianCompositer : public Compositer virtual bool append(const aliceVision::image::Image& color, const aliceVision::image::Image& inputMask, const aliceVision::image::Image& inputWeights, - size_t offset_x, size_t offset_y) + int offset_x, int offset_y) { size_t optimalScale = getOptimalScale(color.Width(), color.Height()); if(optimalScale < _bands) @@ -113,7 +113,6 @@ class LaplacianCompositer : public Compositer [](const image::RGBAfColor & a) -> image::RGBAfColor { image::RGBAfColor out; - out.r() = std::exp(a.r()); out.g() = std::exp(a.g()); out.b() = std::exp(a.b()); diff --git a/src/aliceVision/panorama/laplacianPyramid.cpp b/src/aliceVision/panorama/laplacianPyramid.cpp index a44e630acb..0ef9878c2b 100644 --- a/src/aliceVision/panorama/laplacianPyramid.cpp +++ b/src/aliceVision/panorama/laplacianPyramid.cpp @@ -189,8 +189,6 @@ bool LaplacianPyramid::augment(image::TileCacheManager::shared_ptr & cacheManage _weights.push_back(pyramidWeights); } - std::cout << "ok" << std::endl; - const int processingSize = 512; const size_t borderSize = 5; CachedImage currentImage = largerColor; diff --git a/src/aliceVision/panorama/seams.cpp b/src/aliceVision/panorama/seams.cpp index 6272e7a932..754b74f153 100644 --- a/src/aliceVision/panorama/seams.cpp +++ b/src/aliceVision/panorama/seams.cpp @@ -8,90 +8,171 @@ namespace aliceVision { -void drawBorders(aliceVision::image::Image& inout, aliceVision::image::Image& mask, - size_t offset_x, size_t offset_y) + +bool computeSeamsMap(image::Image& seams, const image::Image& labels) { + if(seams.size() != labels.size()) + { + return false; + } - for(int i = 0; i < mask.Height(); i++) + seams.fill(0); + + for(int j = 1; j < labels.Width() - 1; j++) { - int j = 0; - int di = i + offset_y; - int dj = j + offset_x; - if(dj >= inout.Width()) + IndexT label = labels(0, j); + IndexT same = true; + + same &= (labels(0, j - 1) == label); + same &= (labels(0, j + 1) == label); + same &= (labels(1, j - 1) == label); + same &= (labels(1, j) == label); + same &= (labels(1, j + 1) == label); + + if(same) { - dj = dj - inout.Width(); + continue; } - if(mask(i, j)) + seams(0, j) = 255; + } + + int lastrow = labels.Height() - 1; + for(int j = 1; j < labels.Width() - 1; j++) + { + IndexT label = labels(lastrow, j); + IndexT same = true; + + same &= (labels(lastrow - 1, j - 1) == label); + same &= (labels(lastrow - 1, j + 1) == label); + same &= (labels(lastrow, j - 1) == label); + same &= (labels(lastrow, j) == label); + same &= (labels(lastrow, j + 1) == label); + + if(same) { - inout(di, dj) = image::RGBAfColor(0.0f, 1.0f, 0.0f, 1.0f); + continue; } + + seams(lastrow, j) = 255; } - for(int i = 0; i < mask.Height(); i++) + for(int i = 1; i < labels.Height() - 1; i++) { - int j = mask.Width() - 1; - int di = i + offset_y; - int dj = j + offset_x; - if(dj >= inout.Width()) + + for(int j = 1; j < labels.Width() - 1; j++) { - dj = dj - inout.Width(); + + IndexT label = labels(i, j); + IndexT same = true; + + same &= (labels(i - 1, j - 1) == label); + same &= (labels(i - 1, j) == label); + same &= (labels(i - 1, j + 1) == label); + same &= (labels(i, j - 1) == label); + same &= (labels(i, j + 1) == label); + same &= (labels(i + 1, j - 1) == label); + same &= (labels(i + 1, j) == label); + same &= (labels(i + 1, j + 1) == label); + + if(same) + { + continue; + } + + seams(i, j) = 255; } + } + + return true; +} + +bool drawBorders(CachedImage& inout, const aliceVision::image::Image& mask, size_t offsetX, size_t offsetY) +{ + BoundingBox bb; + bb.left = offsetX; + bb.top = offsetY; + bb.width = mask.Width(); + bb.height = mask.Height(); + + const int border = 2; + BoundingBox dilatedBb = bb.dilate(border); + dilatedBb.clampTop(); + dilatedBb.clampBottom(inout.getHeight() - 1); + int leftBorder = bb.left - dilatedBb.left; + int topBorder = bb.top - dilatedBb.top; + + if (dilatedBb.left < 0) + { + dilatedBb.left += inout.getWidth(); + } + + image::Image extractedColor(dilatedBb.width, dilatedBb.height); + if (!loopyCachedImageExtract(extractedColor, inout, dilatedBb)) + { + return false; + } + + for(int i = 0; i < mask.Height(); i++) + { + int j = 0; + int di = i + topBorder; + int dj = j + leftBorder; if(mask(i, j)) { - inout(di, dj) = image::RGBAfColor(0.0f, 1.0f, 0.0f, 1.0f); + extractedColor(di, dj) = image::RGBAfColor(0.0f, 1.0f, 0.0f, 1.0f); } } - for(int j = 0; j < mask.Width(); j++) + for(int i = 0; i < mask.Height(); i++) { - int i = 0; - int di = i + offset_y; - int dj = j + offset_x; - if(dj >= inout.Width()) + int j = mask.Width() - 1; + int di = i + topBorder; + int dj = j + leftBorder; + + if(mask(i, j)) { - dj = dj - inout.Width(); + extractedColor(di, dj) = image::RGBAfColor(0.0f, 1.0f, 0.0f, 1.0f); } + } + for(int j = 0; j < mask.Width(); j++) + { + int i = 0; + int di = i + topBorder; + int dj = j + leftBorder; + if(mask(i, j)) { - inout(di, dj) = image::RGBAfColor(0.0f, 1.0f, 0.0f, 1.0f); + extractedColor(di, dj) = image::RGBAfColor(0.0f, 1.0f, 0.0f, 1.0f); } } for(int j = 0; j < mask.Width(); j++) { int i = mask.Height() - 1; - int di = i + offset_y; - int dj = j + offset_x; - if(dj >= inout.Width()) - { - dj = dj - inout.Width(); - } + int di = i + topBorder; + int dj = j + leftBorder; if(mask(i, j)) { - inout(di, dj) = image::RGBAfColor(0.0f, 1.0f, 0.0f, 1.0f); + extractedColor(di, dj) = image::RGBAfColor(0.0f, 1.0f, 0.0f, 1.0f); } } for(int i = 1; i < mask.Height() - 1; i++) { - - int di = i + offset_y; + int di = i + topBorder; for(int j = 1; j < mask.Width() - 1; j++) { - - int dj = j + offset_x; - if(dj >= inout.Width()) - { - dj = dj - inout.Width(); - } + int dj = j + leftBorder; if(!mask(i, j)) + { continue; + } unsigned char others = true; others &= mask(i - 1, j - 1); @@ -100,17 +181,95 @@ void drawBorders(aliceVision::image::Image& inout, aliceVisio others &= mask(i, j + 1); others &= mask(i + 1, j - 1); others &= mask(i + 1, j + 1); - if(others) + if(others) { continue; + } - inout(di, dj) = image::RGBAfColor(0.0f, 1.0f, 0.0f, 1.0f); + extractedColor(di, dj) = image::RGBAfColor(0.0f, 1.0f, 0.0f, 1.0f); } } + + BoundingBox inputBb = dilatedBb; + inputBb.left = 0; + inputBb.top = 0; + if (!loopyCachedImageAssign(inout, extractedColor, dilatedBb, inputBb)) + { + return false; + } + + return true; } -void drawSeams(aliceVision::image::Image& inout, aliceVision::image::Image& labels) +bool drawSeams(CachedImage& inout, CachedImage& labels) { + const int processingSize = 512; + + //Process image rectangle by rectangle + for (int y = 0; y < inout.getHeight(); y += processingSize) + { + for (int x = 0; x < inout.getWidth(); x += processingSize) + { + //Compute the initial bouding box for this rectangle + BoundingBox currentBbox; + currentBbox.left = x; + currentBbox.top = y; + currentBbox.width = processingSize; + currentBbox.height = processingSize; + currentBbox.clampLeft(); + currentBbox.clampTop(); + currentBbox.clampRight(inout.getWidth() - 1); + currentBbox.clampBottom(inout.getHeight() - 1); + + image::Image extractedColor(currentBbox.width, currentBbox.height); + if (!loopyCachedImageExtract(extractedColor, inout, currentBbox)) + { + return false; + } + + image::Image extractedLabels(currentBbox.width, currentBbox.height); + if (!loopyCachedImageExtract(extractedLabels, labels, currentBbox)) + { + return false; + } + for(int i = 1; i < extractedLabels.Height() - 1; i++) + { + + for(int j = 1; j < extractedLabels.Width() - 1; j++) + { + + IndexT label = extractedLabels(i, j); + IndexT same = true; + + same &= (extractedLabels(i - 1, j - 1) == label); + same &= (extractedLabels(i - 1, j + 1) == label); + same &= (extractedLabels(i, j - 1) == label); + same &= (extractedLabels(i, j + 1) == label); + same &= (extractedLabels(i + 1, j - 1) == label); + same &= (extractedLabels(i + 1, j + 1) == label); + + if(same) + { + continue; + } + + extractedColor(i, j) = image::RGBAfColor(1.0f, 0.0f, 0.0f, 1.0f); + } + } + + BoundingBox inputBbox = currentBbox; + inputBbox.left = 0; + inputBbox.top = 0; + if (!loopyCachedImageAssign(inout, extractedColor, currentBbox, inputBbox)) + { + return false; + } + } + } + + return true; +} +/* for(int i = 1; i < labels.Height() - 1; i++) { @@ -134,8 +293,7 @@ void drawSeams(aliceVision::image::Image& inout, aliceVision: inout(i, j) = image::RGBAfColor(1.0f, 0.0f, 0.0f, 1.0f); } - } -} + }*/ bool WTASeams::initialize(image::TileCacheManager::shared_ptr & cacheManager) { @@ -431,9 +589,28 @@ bool HierarchicalGraphcutSeams::process() { ALICEVISION_LOG_INFO("Hierachical graphcut processing level #" << level); - if (level < _countLevels - 1) + CachedImage & smallLabels = _graphcuts[level].getLabels(); + int w = smallLabels.getWidth(); + int h = smallLabels.getHeight(); + + if (level == _countLevels - 1) + { + _graphcuts[level].setMaximalDistance(w + h); + } + else { - _graphcuts[level].setMaximalDistance(10); + if (w < 2000) + { + _graphcuts[level].setMaximalDistance(100); + } + else if (w < 5000) + { + _graphcuts[level].setMaximalDistance(10); + } + else + { + _graphcuts[level].setMaximalDistance(3); + } } if(!_graphcuts[level].process()) @@ -447,7 +624,6 @@ bool HierarchicalGraphcutSeams::process() } //Enlarge result of this level to be an initialization for next level - CachedImage & smallLabels = _graphcuts[level].getLabels(); CachedImage & largeLabels = _graphcuts[level - 1].getLabels(); const int processingSize = 256; diff --git a/src/aliceVision/panorama/seams.hpp b/src/aliceVision/panorama/seams.hpp index 5b422bd306..f8c4a756f1 100644 --- a/src/aliceVision/panorama/seams.hpp +++ b/src/aliceVision/panorama/seams.hpp @@ -9,10 +9,9 @@ namespace aliceVision { +bool drawBorders(CachedImage & inout, const aliceVision::image::Image& mask, size_t offsetX, size_t offsetY); -void drawBorders(aliceVision::image::Image& inout, aliceVision::image::Image& mask, - size_t offset_x, size_t offset_y); -void drawSeams(aliceVision::image::Image& inout, aliceVision::image::Image& labels); +bool drawSeams(CachedImage& inout, CachedImage& labels); bool getMaskFromLabels(aliceVision::image::Image & mask, CachedImage & labels, IndexT index, size_t offset_x, size_t offset_y); diff --git a/src/software/pipeline/main_panoramaCompositing.cpp b/src/software/pipeline/main_panoramaCompositing.cpp index a0b94985f7..9420488281 100644 --- a/src/software/pipeline/main_panoramaCompositing.cpp +++ b/src/software/pipeline/main_panoramaCompositing.cpp @@ -77,8 +77,10 @@ bool buildMap(BoundingBoxMap & map, const sfmData::SfMData& sfmData, const std:: return true; } -bool computeWTALabels(CachedImage & labels, image::TileCacheManager::shared_ptr& cacheManager, const sfmData::SfMData& sfmData, const std::string & inputPath, std::pair & panoramaSize) +bool computeWTALabels(CachedImage & labels, image::TileCacheManager::shared_ptr& cacheManager, const std::vector> & views, const std::string & inputPath, std::pair & panoramaSize) { + ALICEVISION_LOG_INFO("Estimating initial labels for panorama"); + WTASeams seams(panoramaSize.first, panoramaSize.second); if (!seams.initialize(cacheManager)) @@ -86,17 +88,13 @@ bool computeWTALabels(CachedImage & labels, image::TileCacheManager::sha return false; } - for(const auto& viewIt : sfmData.getViews()) + for (const auto& viewIt : views) { - if(!sfmData.isPoseAndIntrinsicDefined(viewIt.second.get())) - { - // skip unreconstructed views - continue; - } + IndexT viewId = viewIt->getViewId(); // Load mask - const std::string maskPath = (fs::path(inputPath) / (std::to_string(viewIt.first) + "_mask.exr")).string(); - ALICEVISION_LOG_INFO("Load mask with path " << maskPath); + const std::string maskPath = (fs::path(inputPath) / (std::to_string(viewId) + "_mask.exr")).string(); + ALICEVISION_LOG_TRACE("Load mask with path " << maskPath); image::Image mask; image::readImage(maskPath, mask, image::EImageColorSpace::NO_CONVERSION); @@ -106,12 +104,12 @@ bool computeWTALabels(CachedImage & labels, image::TileCacheManager::sha const std::size_t offsetY = metadata.find("AliceVision:offsetY")->get_int(); // Load Weights - const std::string weightsPath = (fs::path(inputPath) / (std::to_string(viewIt.first) + "_weight.exr")).string(); - ALICEVISION_LOG_INFO("Load weights with path " << weightsPath); + const std::string weightsPath = (fs::path(inputPath) / (std::to_string(viewId) + "_weight.exr")).string(); + ALICEVISION_LOG_TRACE("Load weights with path " << weightsPath); image::Image weights; image::readImage(weightsPath, weights, image::EImageColorSpace::NO_CONVERSION); - if (!seams.append(mask, weights, viewIt.first, offsetX, offsetY)) + if (!seams.append(mask, weights, viewId, offsetX, offsetY)) { return false; } @@ -119,13 +117,12 @@ bool computeWTALabels(CachedImage & labels, image::TileCacheManager::sha labels = seams.getLabels(); - - return true; } -bool computeGCLabels(CachedImage & labels, image::TileCacheManager::shared_ptr& cacheManager, const sfmData::SfMData& sfmData, const std::string & inputPath, std::pair & panoramaSize, int smallestViewScale) +bool computeGCLabels(CachedImage & labels, image::TileCacheManager::shared_ptr& cacheManager, const std::vector> & views, const std::string & inputPath, std::pair & panoramaSize, int smallestViewScale) { + ALICEVISION_LOG_INFO("Estimating smart seams for panorama"); int pyramidSize = std::max(0, smallestViewScale - 1); ALICEVISION_LOG_INFO("Graphcut pyramid size is " << pyramidSize); @@ -142,23 +139,19 @@ bool computeGCLabels(CachedImage & labels, image::TileCacheManager::shar return false; } - for (const auto& viewIt : sfmData.getViews()) + for (const auto& viewIt : views) { - if(!sfmData.isPoseAndIntrinsicDefined(viewIt.second.get())) - { - // skip unreconstructed views - continue; - } + IndexT viewId = viewIt->getViewId(); // Load mask - const std::string maskPath = (fs::path(inputPath) / (std::to_string(viewIt.first) + "_mask.exr")).string(); - ALICEVISION_LOG_INFO("Load mask with path " << maskPath); + const std::string maskPath = (fs::path(inputPath) / (std::to_string(viewId) + "_mask.exr")).string(); + ALICEVISION_LOG_TRACE("Load mask with path " << maskPath); image::Image mask; image::readImage(maskPath, mask, image::EImageColorSpace::NO_CONVERSION); // Load Color - const std::string colorsPath = (fs::path(inputPath) / (std::to_string(viewIt.first) + ".exr")).string(); - ALICEVISION_LOG_INFO("Load colors with path " << colorsPath); + const std::string colorsPath = (fs::path(inputPath) / (std::to_string(viewId) + ".exr")).string(); + ALICEVISION_LOG_TRACE("Load colors with path " << colorsPath); image::Image colors; image::readImage(colorsPath, colors, image::EImageColorSpace::NO_CONVERSION); @@ -169,7 +162,7 @@ bool computeGCLabels(CachedImage & labels, image::TileCacheManager::shar // Append to graph cut - if (!seams.append(colors, mask, viewIt.first, offsetX, offsetY)) + if (!seams.append(colors, mask, viewId, offsetX, offsetY)) { return false; } @@ -194,6 +187,8 @@ int aliceVision_main(int argc, char** argv) std::string compositerType = "multiband"; std::string overlayType = "none"; bool useGraphCut = true; + bool useSeams = false; + bool useWeights = false; bool showBorders = false; bool showSeams = false; @@ -311,18 +306,40 @@ int aliceVision_main(int argc, char** argv) // Configure the cache manager memory cacheManager->setInCoreMaxObjectCount(1000); - LaplacianCompositer compositer(cacheManager, panoramaSize.first, panoramaSize.second, 5); - - if (!compositer.initialize()) + if (overlayType == "borders" || overlayType == "all") { - ALICEVISION_LOG_ERROR("Failed to initialize compositer"); - return EXIT_FAILURE; + showBorders = true; } - BoundingBoxMap map; - if (!buildMap(map, sfmData, warpingFolder)) + if (overlayType == "seams" || overlayType == "all") { + showSeams = true; + } + + std::unique_ptr compositer; + if (compositerType == "multiband") { - ALICEVISION_LOG_ERROR("Error Building map"); + compositer = std::unique_ptr(new LaplacianCompositer(cacheManager, panoramaSize.first, panoramaSize.second, 1)); + useSeams = true; + useWeights = false; + } + else if (compositerType == "alpha") + { + /*compositer = std::unique_ptr(new AlphaCompositer(cacheManager, panoramaSize.first, panoramaSize.second)); + useGraphCut = false; + useSeams = false; + useWeights = true;*/ + } + else + { + /*compositer = std::unique_ptr(new Compositer(cacheManager, panoramaSize.first, panoramaSize.second)); + useGraphCut = false; + useSeams = false; + useWeights = false;*/ + } + + if (!compositer->initialize()) + { + ALICEVISION_LOG_ERROR("Failed to initialize compositer"); return EXIT_FAILURE; } @@ -348,7 +365,7 @@ int aliceVision_main(int argc, char** argv) image::readImage(maskPath, mask, image::EImageColorSpace::NO_CONVERSION); //Estimate scale - size_t scale = compositer.getOptimalScale(mask.Width(), mask.Height()); + size_t scale = compositer->getOptimalScale(mask.Width(), mask.Height()); mapViewsScale[scale].push_back(it.second); } @@ -369,83 +386,126 @@ int aliceVision_main(int argc, char** argv) } } + ALICEVISION_LOG_INFO(viewOrderedByScale.size() << " views to process"); + CachedImage labels; - if (!computeWTALabels(labels, cacheManager, sfmData, warpingFolder, panoramaSize)) + if (useSeams) { - ALICEVISION_LOG_ERROR("Error computing initial labels"); - return EXIT_FAILURE; - } - - labels.writeImage("/home/mmoc/labels_wta.exr"); + if (!computeWTALabels(labels, cacheManager, viewOrderedByScale, warpingFolder, panoramaSize)) + { + ALICEVISION_LOG_ERROR("Error computing initial labels"); + return EXIT_FAILURE; + } - - if (!computeGCLabels(labels, cacheManager, sfmData, warpingFolder, panoramaSize, smallestScale)) - { - ALICEVISION_LOG_ERROR("Error computing graph cut labels"); - return EXIT_FAILURE; + if (useGraphCut) + { + if (!computeGCLabels(labels, cacheManager, viewOrderedByScale, warpingFolder, panoramaSize, smallestScale)) + { + ALICEVISION_LOG_ERROR("Error computing graph cut labels"); + return EXIT_FAILURE; + } + } } - labels.writeImage("/home/mmoc/labels_gc.exr"); - size_t pos = 0; - - std::cout << "compositing" << std::endl; - for(const auto & view : viewOrderedByScale) { IndexT viewId = view->getViewId(); - if(!sfmData.isPoseAndIntrinsicDefined(view.get())) - { - // skip unreconstructed views - continue; - } pos++; + ALICEVISION_LOG_INFO("Processing input " << pos << "/" << viewOrderedByScale.size()); // Load image and convert it to linear colorspace const std::string imagePath = (fs::path(warpingFolder) / (std::to_string(viewId) + ".exr")).string(); - //ALICEVISION_LOG_INFO("Load image with path " << imagePath); + ALICEVISION_LOG_TRACE("Load image with path " << imagePath); image::Image source; image::readImage(imagePath, source, image::EImageColorSpace::NO_CONVERSION); + // Retrieve position of image in panorama oiio::ParamValueList metadata = image::readImageMetadata(imagePath); const std::size_t offsetX = metadata.find("AliceVision:offsetX")->get_int(); const std::size_t offsetY = metadata.find("AliceVision:offsetY")->get_int(); // Load mask const std::string maskPath = (fs::path(warpingFolder) / (std::to_string(viewId) + "_mask.exr")).string(); - - //ALICEVISION_LOG_INFO("Load mask with path " << maskPath); + ALICEVISION_LOG_TRACE("Load mask with path " << maskPath); image::Image mask; image::readImage(maskPath, mask, image::EImageColorSpace::NO_CONVERSION); // Load Weights - /*const std::string weightsPath = (fs::path(warpingFolder) / (std::to_string(viewId) + "_weight.exr")).string(); - ALICEVISION_LOG_INFO("Load weights with path " << weightsPath); image::Image weights; - image::readImage(weightsPath, weights, image::EImageColorSpace::NO_CONVERSION);*/ + if (useWeights) + { + const std::string weightsPath = (fs::path(warpingFolder) / (std::to_string(viewId) + "_weight.exr")).string(); + ALICEVISION_LOG_TRACE("Load weights with path " << weightsPath); + image::readImage(weightsPath, weights, image::EImageColorSpace::NO_CONVERSION); + } + else if (useSeams) + { + weights = image::Image(mask.Width(), mask.Height()); + if (!getMaskFromLabels(weights, labels, viewId, offsetX, offsetY)) + { + ALICEVISION_LOG_ERROR("Error estimating seams image"); + return EXIT_FAILURE; + } + } - - image::Image seams(mask.Width(), mask.Height()); - if (!getMaskFromLabels(seams, labels, viewId, offsetX, offsetY)) + if (!compositer->append(source, mask, weights, offsetX, offsetY)) { - ALICEVISION_LOG_ERROR("Error estimating seams image"); return EXIT_FAILURE; } + } + + ALICEVISION_LOG_INFO("Final processing"); + if (!compositer->terminate()) + { + ALICEVISION_LOG_ERROR("Error terminating panorama"); + return EXIT_FAILURE; + } + + if (showBorders) + { + ALICEVISION_LOG_INFO("Drawing borders on panorama"); + + for(const auto & view : viewOrderedByScale) + { + IndexT viewId = view->getViewId(); + + // Load mask + const std::string maskPath = (fs::path(warpingFolder) / (std::to_string(viewId) + "_mask.exr")).string(); + ALICEVISION_LOG_TRACE("Load mask with path " << maskPath); + image::Image mask; + image::readImage(maskPath, mask, image::EImageColorSpace::NO_CONVERSION); + + oiio::ParamValueList metadata = image::readImageMetadata(maskPath); + const std::size_t offsetX = metadata.find("AliceVision:offsetX")->get_int(); + const std::size_t offsetY = metadata.find("AliceVision:offsetY")->get_int(); + + if (!compositer->drawBorders(mask, offsetX, offsetY)) + { + ALICEVISION_LOG_ERROR("Failed to draw borders"); + return EXIT_FAILURE; + } + } + } - std::cout << pos << std::endl; - if (!compositer.append(source, mask, seams, offsetX, offsetY)) + if (showSeams && useSeams) + { + ALICEVISION_LOG_INFO("Drawing seams on panorama"); + + if (!compositer->drawSeams(labels)) { + ALICEVISION_LOG_ERROR("Failed to draw borders"); return EXIT_FAILURE; } } - if (!compositer.terminate()) + ALICEVISION_LOG_INFO("Saving panorama to file"); + if (!compositer->save(outputPanorama)) { + ALICEVISION_LOG_ERROR("Impossible to save file"); return EXIT_FAILURE; } - compositer.save(outputPanorama); - return EXIT_SUCCESS; } From 690af8c14f9770d3c931f1b6e1786add0e642ce4 Mon Sep 17 00:00:00 2001 From: Fabien Servant Date: Thu, 22 Oct 2020 18:17:24 +0200 Subject: [PATCH 24/79] [panorama] removing non portable stuff --- src/aliceVision/panorama/boundingBox.hpp | 6 +- .../pipeline/main_panoramaWarping.cpp | 82 +++++++++---------- 2 files changed, 43 insertions(+), 45 deletions(-) diff --git a/src/aliceVision/panorama/boundingBox.hpp b/src/aliceVision/panorama/boundingBox.hpp index e81b58ff39..c3110cdd1c 100644 --- a/src/aliceVision/panorama/boundingBox.hpp +++ b/src/aliceVision/panorama/boundingBox.hpp @@ -32,15 +32,15 @@ struct BoundingBox { } - constexpr int getRight() const { + int getRight() const { return left + width - 1; } - constexpr int getBottom() const { + int getBottom() const { return top + height - 1; } - constexpr bool isEmpty() const { + bool isEmpty() const { return (width <= 0 || height <= 0); } diff --git a/src/software/pipeline/main_panoramaWarping.cpp b/src/software/pipeline/main_panoramaWarping.cpp index e6aeb3aade..3c204883c9 100644 --- a/src/software/pipeline/main_panoramaWarping.cpp +++ b/src/software/pipeline/main_panoramaWarping.cpp @@ -407,50 +407,48 @@ int aliceVision_main(int argc, char** argv) #pragma omp parallel for + for (int boxId = 0; boxId < boxes.size(); boxId++) { - for (int boxId = 0; boxId < boxes.size(); boxId++) + BoundingBox localBbox = boxes[boxId]; + + int x = localBbox.left - globalBbox.left; + int y = localBbox.top - globalBbox.top; + + // Prepare coordinates map + CoordinatesMap map; + if (!map.build(panoramaSize, camPose, *(intrinsic.get()), localBbox)) + { + continue; + } + + // Warp image + GaussianWarper warper; + if (!warper.warp(map, pyramid)) { + continue; + } + + // Alpha mask + aliceVision::image::Image weights; + if (!distanceToCenter(weights, map, intrinsic->w(), intrinsic->h())) { + continue; + } + + // Store + #pragma omp critical + { + out_view->write_tile(x, y, 0, oiio::TypeDesc::FLOAT, warper.getColor().data()); + } + + // Store + #pragma omp critical + { + out_mask->write_tile(x, y, 0, oiio::TypeDesc::UCHAR, warper.getMask().data()); + } + + // Store + #pragma omp critical { - BoundingBox localBbox = boxes[boxId]; - - int x = localBbox.left - globalBbox.left; - int y = localBbox.top - globalBbox.top; - - // Prepare coordinates map - CoordinatesMap map; - if (!map.build(panoramaSize, camPose, *(intrinsic.get()), localBbox)) - { - continue; - } - - // Warp image - GaussianWarper warper; - if (!warper.warp(map, pyramid)) { - continue; - } - - // Alpha mask - aliceVision::image::Image weights; - if (!distanceToCenter(weights, map, intrinsic->w(), intrinsic->h())) { - continue; - } - - // Store - #pragma omp critical - { - out_view->write_tile(x, y, 0, oiio::TypeDesc::FLOAT, warper.getColor().data()); - } - - // Store - #pragma omp critical - { - out_mask->write_tile(x, y, 0, oiio::TypeDesc::UCHAR, warper.getMask().data()); - } - - // Store - #pragma omp critical - { - out_weights->write_tile(x, y, 0, oiio::TypeDesc::FLOAT, weights.data()); - } + out_weights->write_tile(x, y, 0, oiio::TypeDesc::FLOAT, weights.data()); } } From e581d5d23de371c67de21cd99fe395ed6d46af3b Mon Sep 17 00:00:00 2001 From: Fabien Servant Date: Thu, 22 Oct 2020 18:58:43 +0200 Subject: [PATCH 25/79] [panorama] use 16 bits float option --- src/aliceVision/panorama/cachedImage.cpp | 27 ++++++++++++++----- src/aliceVision/panorama/cachedImage.hpp | 12 ++++----- src/aliceVision/panorama/compositer.hpp | 22 +++++++++++++-- src/aliceVision/panorama/imageOps.hpp | 1 + .../pipeline/main_panoramaCompositing.cpp | 10 +++---- .../pipeline/main_panoramaWarping.cpp | 9 ++++++- 6 files changed, 60 insertions(+), 21 deletions(-) diff --git a/src/aliceVision/panorama/cachedImage.cpp b/src/aliceVision/panorama/cachedImage.cpp index 2baea431be..5b8fea3152 100644 --- a/src/aliceVision/panorama/cachedImage.cpp +++ b/src/aliceVision/panorama/cachedImage.cpp @@ -4,7 +4,7 @@ namespace aliceVision { template <> -bool CachedImage::writeImage(const std::string& path) +bool CachedImage::writeImage(const std::string& path, const image::EStorageDataType &storageDataType) { std::unique_ptr out = oiio::ImageOutput::create(path); @@ -13,7 +13,14 @@ bool CachedImage::writeImage(const std::string& path) return false; } - oiio::ImageSpec spec(_width, _height, 4, oiio::TypeDesc::FLOAT); + oiio::TypeDesc typeColor = oiio::TypeDesc::FLOAT; + if (storageDataType == image::EStorageDataType::Half || storageDataType == image::EStorageDataType::HalfFinite) + { + typeColor = oiio::TypeDesc::HALF; + } + + + oiio::ImageSpec spec(_width, _height, 4, typeColor); spec.tile_width = _tileSize; spec.tile_height = _tileSize; @@ -47,7 +54,7 @@ bool CachedImage::writeImage(const std::string& path) } template <> -bool CachedImage::writeImage(const std::string& path) +bool CachedImage::writeImage(const std::string& path, const image::EStorageDataType &storageDataType) { std::unique_ptr out = oiio::ImageOutput::create(path); @@ -56,7 +63,13 @@ bool CachedImage::writeImage(const std::string& path) return false; } - oiio::ImageSpec spec(_width, _height, 3, oiio::TypeDesc::FLOAT); + oiio::TypeDesc typeColor = oiio::TypeDesc::FLOAT; + if (storageDataType == image::EStorageDataType::Half || storageDataType == image::EStorageDataType::HalfFinite) + { + typeColor = oiio::TypeDesc::HALF; + } + + oiio::ImageSpec spec(_width, _height, 3, typeColor); spec.tile_width = _tileSize; spec.tile_height = _tileSize; @@ -90,7 +103,7 @@ bool CachedImage::writeImage(const std::string& path) } template <> -bool CachedImage::writeImage(const std::string& path) +bool CachedImage::writeImage(const std::string& path, const image::EStorageDataType &storageDataType) { std::unique_ptr out = oiio::ImageOutput::create(path); @@ -133,7 +146,7 @@ bool CachedImage::writeImage(const std::string& path) } template <> -bool CachedImage::writeImage(const std::string& path) +bool CachedImage::writeImage(const std::string& path, const image::EStorageDataType &storageDataType) { std::unique_ptr out = oiio::ImageOutput::create(path); @@ -176,7 +189,7 @@ bool CachedImage::writeImage(const std::string& path) } template <> -bool CachedImage::writeImage(const std::string& path) +bool CachedImage::writeImage(const std::string& path, const image::EStorageDataType &storageDataType) { std::unique_ptr out = oiio::ImageOutput::create(path); diff --git a/src/aliceVision/panorama/cachedImage.hpp b/src/aliceVision/panorama/cachedImage.hpp index 9e30b3887c..55219d179c 100644 --- a/src/aliceVision/panorama/cachedImage.hpp +++ b/src/aliceVision/panorama/cachedImage.hpp @@ -68,7 +68,7 @@ class CachedImage return true; } - bool writeImage(const std::string& path) + bool writeImage(const std::string& path, const image::EStorageDataType &storageDataType = image::EStorageDataType::Auto) { ALICEVISION_LOG_ERROR("incorrect template function"); @@ -484,18 +484,18 @@ class CachedImage }; template <> -bool CachedImage::writeImage(const std::string& path); +bool CachedImage::writeImage(const std::string& path, const image::EStorageDataType &storageDataType); template <> -bool CachedImage::writeImage(const std::string& path); +bool CachedImage::writeImage(const std::string& path, const image::EStorageDataType &storageDataType); template <> -bool CachedImage::writeImage(const std::string& path); +bool CachedImage::writeImage(const std::string& path, const image::EStorageDataType &storageDataType); template <> -bool CachedImage::writeImage(const std::string& path); +bool CachedImage::writeImage(const std::string& path, const image::EStorageDataType &storageDataType); template <> -bool CachedImage::writeImage(const std::string& path); +bool CachedImage::writeImage(const std::string& path, const image::EStorageDataType &storageDataType); } // namespace aliceVision \ No newline at end of file diff --git a/src/aliceVision/panorama/compositer.hpp b/src/aliceVision/panorama/compositer.hpp index 16b6e9a3c9..b81c15b8dc 100644 --- a/src/aliceVision/panorama/compositer.hpp +++ b/src/aliceVision/panorama/compositer.hpp @@ -6,6 +6,7 @@ #include "imageOps.hpp" #include "seams.hpp" +use namespace aliceVision { @@ -90,9 +91,26 @@ class Compositer virtual bool terminate() { return true; } - bool save(const std::string &path) + bool save(const std::string &path, const image::EStorageDataType &storageDataType) { - if(!_panorama.writeImage(path)) + if (storageDataType == image::EStorageDataType::HalfFinite) + { + _panorama.perPixelOperation([](const image::RGBAfColor & c) + { + image::RGBAfColor ret; + + const float limit = float(HALF_MAX); + + ret.r() = clamp(c.r(), -limit, limit); + ret.g() = clamp(c.g(), -limit, limit); + ret.b() = clamp(c.b(), -limit, limit); + ret.a() = c.a(); + + return ret; + }); + } + + if(!_panorama.writeImage(path, storageDataType)) { return false; } diff --git a/src/aliceVision/panorama/imageOps.hpp b/src/aliceVision/panorama/imageOps.hpp index a42310b728..f1f90b9a08 100644 --- a/src/aliceVision/panorama/imageOps.hpp +++ b/src/aliceVision/panorama/imageOps.hpp @@ -2,6 +2,7 @@ #include #include "cachedImage.hpp" +#include namespace aliceVision { diff --git a/src/software/pipeline/main_panoramaCompositing.cpp b/src/software/pipeline/main_panoramaCompositing.cpp index 9420488281..af1cea53d8 100644 --- a/src/software/pipeline/main_panoramaCompositing.cpp +++ b/src/software/pipeline/main_panoramaCompositing.cpp @@ -324,17 +324,17 @@ int aliceVision_main(int argc, char** argv) } else if (compositerType == "alpha") { - /*compositer = std::unique_ptr(new AlphaCompositer(cacheManager, panoramaSize.first, panoramaSize.second)); + compositer = std::unique_ptr(new AlphaCompositer(cacheManager, panoramaSize.first, panoramaSize.second)); useGraphCut = false; useSeams = false; - useWeights = true;*/ + useWeights = true; } else { - /*compositer = std::unique_ptr(new Compositer(cacheManager, panoramaSize.first, panoramaSize.second)); + compositer = std::unique_ptr(new Compositer(cacheManager, panoramaSize.first, panoramaSize.second)); useGraphCut = false; useSeams = false; - useWeights = false;*/ + useWeights = false; } if (!compositer->initialize()) @@ -501,7 +501,7 @@ int aliceVision_main(int argc, char** argv) } ALICEVISION_LOG_INFO("Saving panorama to file"); - if (!compositer->save(outputPanorama)) + if (!compositer->save(outputPanorama, storageDataType)) { ALICEVISION_LOG_ERROR("Impossible to save file"); return EXIT_FAILURE; diff --git a/src/software/pipeline/main_panoramaWarping.cpp b/src/software/pipeline/main_panoramaWarping.cpp index 3c204883c9..0e74fd3910 100644 --- a/src/software/pipeline/main_panoramaWarping.cpp +++ b/src/software/pipeline/main_panoramaWarping.cpp @@ -169,6 +169,12 @@ int aliceVision_main(int argc, char** argv) // Set verbose level given command line system::Logger::get()->setLogLevel(verboseLevel); + oiio::TypeDesc typeColor = oiio::TypeDesc::FLOAT; + if (storageDataType == image::EStorageDataType::Half || storageDataType == image::EStorageDataType::HalfFinite) + { + typeColor = oiio::TypeDesc::HALF; + } + // Load information about inputs // Camera images // Camera intrinsics @@ -363,8 +369,9 @@ int aliceVision_main(int argc, char** argv) std::unique_ptr out_mask = oiio::ImageOutput::create(maskFilepath); std::unique_ptr out_weights = oiio::ImageOutput::create(weightFilepath); + // Define output properties - oiio::ImageSpec spec_view(globalBbox.width, globalBbox.height, 3, (storageDataType == image::EStorageDataType::Half)?oiio::TypeDesc::HALF:oiio::TypeDesc::FLOAT); + oiio::ImageSpec spec_view(globalBbox.width, globalBbox.height, 3, typeColor); oiio::ImageSpec spec_mask(globalBbox.width, globalBbox.height, 1, oiio::TypeDesc::UCHAR); oiio::ImageSpec spec_weights(globalBbox.width, globalBbox.height, 1, oiio::TypeDesc::HALF); From 7efe93bf89bfd71c9746961e4a695ba26b59844e Mon Sep 17 00:00:00 2001 From: Fabien Servant Date: Fri, 23 Oct 2020 14:47:42 +0200 Subject: [PATCH 26/79] [panorama] some updates on bounding boxes --- src/aliceVision/panorama/CMakeLists.txt | 4 +- src/aliceVision/panorama/boundingBox.hpp | 10 +- src/aliceVision/panorama/boundingBoxMap.cpp | 52 ------ src/aliceVision/panorama/boundingBoxMap.hpp | 24 --- .../panorama/boundingBoxPanoramaMap.cpp | 167 ++++++++++++++++++ .../panorama/boundingBoxPanoramaMap.hpp | 37 ++++ src/aliceVision/panorama/compositer.hpp | 8 +- .../panorama/laplacianCompositer.hpp | 23 ++- src/aliceVision/panorama/seams.cpp | 25 --- src/aliceVision/panorama/seams.hpp | 1 - .../pipeline/main_panoramaCompositing.cpp | 50 ++++-- 11 files changed, 268 insertions(+), 133 deletions(-) delete mode 100644 src/aliceVision/panorama/boundingBoxMap.cpp delete mode 100644 src/aliceVision/panorama/boundingBoxMap.hpp create mode 100644 src/aliceVision/panorama/boundingBoxPanoramaMap.cpp create mode 100644 src/aliceVision/panorama/boundingBoxPanoramaMap.hpp diff --git a/src/aliceVision/panorama/CMakeLists.txt b/src/aliceVision/panorama/CMakeLists.txt index 2d5798685d..aa7d2fa4bf 100644 --- a/src/aliceVision/panorama/CMakeLists.txt +++ b/src/aliceVision/panorama/CMakeLists.txt @@ -2,7 +2,7 @@ set(panorama_files_headers alphaCompositer.hpp boundingBox.hpp - boundingBoxMap.hpp + boundingBoxPanoramaMap.hpp compositer.hpp coordinatesMap.hpp distance.hpp @@ -32,7 +32,7 @@ set(panorama_files_sources seams.cpp imageOps.cpp cachedImage.cpp - boundingBoxMap.cpp + boundingBoxPanoramaMap.cpp ) alicevision_add_library(aliceVision_panorama diff --git a/src/aliceVision/panorama/boundingBox.hpp b/src/aliceVision/panorama/boundingBox.hpp index c3110cdd1c..f7c7fbd1fb 100644 --- a/src/aliceVision/panorama/boundingBox.hpp +++ b/src/aliceVision/panorama/boundingBox.hpp @@ -44,6 +44,10 @@ struct BoundingBox return (width <= 0 || height <= 0); } + int area() const { + return width * height; + } + void snapToGrid(uint32_t gridSize) { @@ -167,7 +171,7 @@ struct BoundingBox } } - BoundingBox doubleSize() + BoundingBox doubleSize() const { BoundingBox b; @@ -179,7 +183,7 @@ struct BoundingBox return b; } - BoundingBox multiply(int factor) + BoundingBox multiply(int factor) const { BoundingBox b; @@ -191,7 +195,7 @@ struct BoundingBox return b; } - BoundingBox divide(int factor) + BoundingBox divide(int factor) const { BoundingBox b; diff --git a/src/aliceVision/panorama/boundingBoxMap.cpp b/src/aliceVision/panorama/boundingBoxMap.cpp deleted file mode 100644 index 0e785d7e8f..0000000000 --- a/src/aliceVision/panorama/boundingBoxMap.cpp +++ /dev/null @@ -1,52 +0,0 @@ -#include "boundingBoxMap.hpp" - -#include - -namespace aliceVision -{ - -bool BoundingBoxMap::append(IndexT index, const BoundingBox & box) -{ - _map[index] = box; - - return true; -} - -bool BoundingBoxMap::getIntersectionList(std::map & intersections, IndexT reference) const -{ - auto it = _map.find(reference); - - if (it == _map.end()) - { - return false; - } - - intersections.clear(); - - BoundingBox refBB = it->second; - if (refBB.isEmpty()) - { - return false; - } - - for (auto item : _map) - { - if (item.first == reference) - { - continue; - } - - BoundingBox intersection = refBB.intersectionWith(item.second); - if (intersection.isEmpty()) - { - continue; - } - - intersections[item.first] = intersection; - } - - return true; -} - - -} \ No newline at end of file diff --git a/src/aliceVision/panorama/boundingBoxMap.hpp b/src/aliceVision/panorama/boundingBoxMap.hpp deleted file mode 100644 index ca56676b84..0000000000 --- a/src/aliceVision/panorama/boundingBoxMap.hpp +++ /dev/null @@ -1,24 +0,0 @@ -#pragma once - -#include "boundingBox.hpp" - -#include -#include - -namespace aliceVision -{ - -class BoundingBoxMap -{ -public: - BoundingBoxMap() = default; - - bool append(IndexT index, const BoundingBox & box); - - bool getIntersectionList(std::map & intersections, IndexT reference) const; - -private: - std::map _map; -}; - -} \ No newline at end of file diff --git a/src/aliceVision/panorama/boundingBoxPanoramaMap.cpp b/src/aliceVision/panorama/boundingBoxPanoramaMap.cpp new file mode 100644 index 0000000000..53ba6f8ff6 --- /dev/null +++ b/src/aliceVision/panorama/boundingBoxPanoramaMap.cpp @@ -0,0 +1,167 @@ +#include "boundingBoxPanoramaMap.hpp" + +#include +#include + +namespace aliceVision +{ + + + +bool BoundingBoxPanoramaMap::append(IndexT index, const BoundingBox & box, int maxScale, int borderSize) +{ + int maxFactor = pow(2, maxScale); + + BoundingBox scaled = box.divide(maxFactor); + + BoundingBox scaledWithBorders = scaled.dilate(borderSize); + + _map[index] = scaledWithBorders.multiply(maxFactor); + + return true; +} + +bool BoundingBoxPanoramaMap::intersect(const BoundingBox & box1, const BoundingBox & box2) const +{ + BoundingBox otherBbox = box2; + BoundingBox otherBboxLoop = box2; + otherBboxLoop.left = otherBbox.left - _panoramaWidth; + BoundingBox otherBboxLoopRight = box2; + otherBboxLoopRight.left = otherBbox.left + _panoramaWidth; + + if (!box1.intersectionWith(otherBbox).isEmpty()) + { + return true; + } + + if (!box1.intersectionWith(otherBboxLoop).isEmpty()) + { + return true; + } + + if (!box1.intersectionWith(otherBboxLoopRight).isEmpty()) + { + return true; + } + + return false; +} + +bool BoundingBoxPanoramaMap::isValid(const std::list & stack) const +{ + if (stack.size() == 0) + { + return false; + } + + if (stack.size() == 1) + { + return true; + } + + auto & item = stack.back(); + IndexT top = *(item.second); + BoundingBox topBb = _map.at(top); + + for (auto it : stack) + { + IndexT current = *(it.second); + if (current == top) continue; + + BoundingBox currentBb = _map.at(current); + + if (intersect(topBb, currentBb)) + { + return false; + } + } + + return true; +} + +bool BoundingBoxPanoramaMap::getBestInitial(std::list & best_items) const +{ + + //For each view, get the list of non overlapping views + //with an id which is superior to them + std::set initialList; + std::map> nonOverlapPerBox; + for (auto it = _map.begin(); it != _map.end(); it++) + { + std::set listNonOverlap; + + initialList.insert(it->first); + + for (auto nextIt = std::next(it); nextIt != _map.end(); nextIt++) + { + if (intersect(it->second, nextIt->second)) + { + continue; + } + + listNonOverlap.insert(nextIt->first); + } + + nonOverlapPerBox[it->first] = listNonOverlap; + } + nonOverlapPerBox[UndefinedIndexT] = initialList; + + size_t best_score = 0; + best_items.clear(); + std::list stack; + stack.push_back(std::make_pair(UndefinedIndexT, nonOverlapPerBox.at(UndefinedIndexT).begin())); + + while (!stack.empty()) + { + if (isValid(stack)) + { + std::list items; + size_t score = 0; + for (auto it : stack) + { + IndexT item; + items.push_back(item); + BoundingBox box = _map.at(item); + score += box.area(); + if (score > best_score) + { + best_items = items; + best_score = score; + } + } + + IndexT id = *stack.back().second; + std::set & addList = nonOverlapPerBox.at(id); + + if (!addList.empty()) + { + stack.push_back(std::make_pair(id, addList.begin())); + continue; + } + } + + while (!stack.empty()) + { + //Iterate over the last list + stackItem & lastItem = stack.back(); + uniqueIndices & list = nonOverlapPerBox.at(lastItem.first); + lastItem.second++; + + //If we reached the end of the list + //Pop back this list + if (stack.back().second == list.end()) + { + stack.pop_back(); + } + else + { + break; + } + } + } + + return true; +} + + +} \ No newline at end of file diff --git a/src/aliceVision/panorama/boundingBoxPanoramaMap.hpp b/src/aliceVision/panorama/boundingBoxPanoramaMap.hpp new file mode 100644 index 0000000000..ad2d880a32 --- /dev/null +++ b/src/aliceVision/panorama/boundingBoxPanoramaMap.hpp @@ -0,0 +1,37 @@ +#pragma once + +#include "boundingBox.hpp" + +#include + +#include +#include + +namespace aliceVision +{ + +class BoundingBoxPanoramaMap +{ +public: + using uniqueIndices = std::set; + using stackItem = std::pair; +public: + BoundingBoxPanoramaMap(int width, int height) : _panoramaWidth(width), _panoramaHeight(height) + { + + } + + bool append(IndexT index, const BoundingBox & box, int maxScale, int borderSize); + bool getBestInitial(std::list & best_items) const; + +private: + bool intersect(const BoundingBox & box1, const BoundingBox & box2) const; + bool isValid(const std::list & stack) const; + +private: + std::map _map; + int _panoramaWidth; + int _panoramaHeight; +}; + +} \ No newline at end of file diff --git a/src/aliceVision/panorama/compositer.hpp b/src/aliceVision/panorama/compositer.hpp index b81c15b8dc..fa1a730983 100644 --- a/src/aliceVision/panorama/compositer.hpp +++ b/src/aliceVision/panorama/compositer.hpp @@ -6,7 +6,6 @@ #include "imageOps.hpp" #include "seams.hpp" -use namespace aliceVision { @@ -118,11 +117,16 @@ class Compositer return true; } - virtual size_t getOptimalScale(int width, int height) + virtual size_t getOptimalScale(int width, int height) const { return 1; } + virtual int getBorderSize() const + { + return 0; + } + bool drawBorders(const aliceVision::image::Image& mask, size_t offsetX, size_t offsetY) { return ::aliceVision::drawBorders(_panorama, mask, offsetX, offsetY); diff --git a/src/aliceVision/panorama/laplacianCompositer.hpp b/src/aliceVision/panorama/laplacianCompositer.hpp index ea728eb049..0134215228 100644 --- a/src/aliceVision/panorama/laplacianCompositer.hpp +++ b/src/aliceVision/panorama/laplacianCompositer.hpp @@ -27,7 +27,7 @@ class LaplacianCompositer : public Compositer return _pyramidPanorama.initialize(_cacheManager); } - virtual size_t getOptimalScale(int width, int height) + virtual size_t getOptimalScale(int width, int height) const { /* Look for the smallest scale such that the image is not smaller than the @@ -38,11 +38,17 @@ class LaplacianCompositer : public Compositer */ size_t minsize = std::min(width, height); - const float gaussian_filter_size = 5.0f; - size_t optimal_scale = size_t(floor(std::log2(double(minsize) / gaussian_filter_size))); + + int gaussianFilterSize = 1 + 2 * _gaussianFilterRadius; + + size_t optimal_scale = size_t(floor(std::log2(double(minsize) / gaussianFilterSize))); return optimal_scale; } + virtual int getBorderSize() const + { + return _gaussianFilterRadius; + } virtual bool append(const aliceVision::image::Image& color, const aliceVision::image::Image& inputMask, @@ -50,17 +56,19 @@ class LaplacianCompositer : public Compositer int offset_x, int offset_y) { size_t optimalScale = getOptimalScale(color.Width(), color.Height()); - if(optimalScale < _bands) + size_t optimalLevelsCount = optimalScale + 1; + + if(optimalLevelsCount < _bands) { - ALICEVISION_LOG_ERROR("Decreasing scale !"); + ALICEVISION_LOG_ERROR("Decreasing level count !"); return false; } //If the input scale is more important than previously processed, // The pyramid must be deepened accordingly - if(optimalScale > _bands) + if(optimalLevelsCount > _bands) { - _bands = optimalScale; + _bands = optimalLevelsCount; _pyramidPanorama.augment(_cacheManager, _bands); } @@ -126,6 +134,7 @@ class LaplacianCompositer : public Compositer } protected: + const float _gaussianFilterRadius = 2.0f; LaplacianPyramid _pyramidPanorama; size_t _bands; }; diff --git a/src/aliceVision/panorama/seams.cpp b/src/aliceVision/panorama/seams.cpp index 754b74f153..ced5c0dbee 100644 --- a/src/aliceVision/panorama/seams.cpp +++ b/src/aliceVision/panorama/seams.cpp @@ -269,31 +269,6 @@ bool drawSeams(CachedImage& inout, CachedImage& label return true; } -/* - for(int i = 1; i < labels.Height() - 1; i++) - { - - for(int j = 1; j < labels.Width() - 1; j++) - { - - IndexT label = labels(i, j); - IndexT same = true; - - same &= (labels(i - 1, j - 1) == label); - same &= (labels(i - 1, j + 1) == label); - same &= (labels(i, j - 1) == label); - same &= (labels(i, j + 1) == label); - same &= (labels(i + 1, j - 1) == label); - same &= (labels(i + 1, j + 1) == label); - - if(same) - { - continue; - } - - inout(i, j) = image::RGBAfColor(1.0f, 0.0f, 0.0f, 1.0f); - } - }*/ bool WTASeams::initialize(image::TileCacheManager::shared_ptr & cacheManager) { diff --git a/src/aliceVision/panorama/seams.hpp b/src/aliceVision/panorama/seams.hpp index f8c4a756f1..d78f955b71 100644 --- a/src/aliceVision/panorama/seams.hpp +++ b/src/aliceVision/panorama/seams.hpp @@ -5,7 +5,6 @@ #include "cachedImage.hpp" #include "graphcut.hpp" -#include "boundingBoxMap.hpp" namespace aliceVision { diff --git a/src/software/pipeline/main_panoramaCompositing.cpp b/src/software/pipeline/main_panoramaCompositing.cpp index af1cea53d8..494a478bc9 100644 --- a/src/software/pipeline/main_panoramaCompositing.cpp +++ b/src/software/pipeline/main_panoramaCompositing.cpp @@ -32,7 +32,7 @@ #include #include #include -#include +#include // These constants define the current software version. // They must be updated when the command line is changed. @@ -45,34 +45,35 @@ namespace po = boost::program_options; namespace bpt = boost::property_tree; namespace fs = boost::filesystem; -bool buildMap(BoundingBoxMap & map, const sfmData::SfMData& sfmData, const std::string & inputPath) +bool buildMap(BoundingBoxPanoramaMap & map, const std::vector> & views, const std::string & inputPath, const std::unique_ptr & compositer) { - /*for (const auto& viewIt : sfmData.getViews()) + for (const auto& viewIt : views) { - if(!sfmData.isPoseAndIntrinsicDefined(viewIt.second.get())) - { - // skip unreconstructed views - continue; - } - // Load mask - const std::string maskPath = (fs::path(inputPath) / (std::to_string(viewIt.first) + "_mask.exr")).string(); + const std::string maskPath = (fs::path(inputPath) / (std::to_string(viewIt->getViewId()) + "_mask.exr")).string(); + ALICEVISION_LOG_TRACE("Load mask with path " << maskPath); + image::Image mask; + image::readImage(maskPath, mask, image::EImageColorSpace::NO_CONVERSION); + oiio::ParamValueList metadata = image::readImageMetadata(maskPath); const std::size_t offsetX = metadata.find("AliceVision:offsetX")->get_int(); const std::size_t offsetY = metadata.find("AliceVision:offsetY")->get_int(); BoundingBox bb; - bb.left = offsetX + contentX; - bb.top = offsetY + contentY; - bb.width = contentW; - bb.height = contentH; + bb.left = offsetX; + bb.top = offsetY; + bb.width = mask.Width(); + bb.height = mask.Height(); + + int border = compositer->getBorderSize(); + int scale = compositer->getOptimalScale(mask.Width(), mask.Height()); - if (!map.append(viewIt.first, bb)) + if (!map.append(viewIt->getViewId(), bb, border, scale)) { return false; } - }*/ + } return true; } @@ -336,6 +337,7 @@ int aliceVision_main(int argc, char** argv) useSeams = false; useWeights = false; } + if (!compositer->initialize()) { @@ -344,7 +346,7 @@ int aliceVision_main(int argc, char** argv) } //Get a list of views ordered by their image scale - size_t smallestScale = 1; + size_t smallestScale = 0; std::vector> viewOrderedByScale; { std::map>> mapViewsScale; @@ -388,6 +390,20 @@ int aliceVision_main(int argc, char** argv) ALICEVISION_LOG_INFO(viewOrderedByScale.size() << " views to process"); + BoundingBoxPanoramaMap map(panoramaSize.first, panoramaSize.second); + if (!buildMap(map, viewOrderedByScale, warpingFolder, compositer)) + { + ALICEVISION_LOG_ERROR("Error computing map"); + return EXIT_FAILURE; + } + + std::list bestInitials; + if (!map.getBestInitial(bestInitials)) + { + ALICEVISION_LOG_ERROR("Error computing best elements"); + return EXIT_SUCCESS; + } + CachedImage labels; if (useSeams) { From 8ca2ca14498cc5a41b4318bb3e012b9e8f822b31 Mon Sep 17 00:00:00 2001 From: Fabien Servant Date: Fri, 23 Oct 2020 17:56:46 +0200 Subject: [PATCH 27/79] [panorama] cleanup stuff --- .../panorama/laplacianCompositer.hpp | 47 ++- src/aliceVision/panorama/laplacianPyramid.cpp | 292 ++---------------- src/aliceVision/panorama/laplacianPyramid.hpp | 15 +- .../pipeline/main_panoramaCompositing.cpp | 4 +- 4 files changed, 76 insertions(+), 282 deletions(-) diff --git a/src/aliceVision/panorama/laplacianCompositer.hpp b/src/aliceVision/panorama/laplacianCompositer.hpp index 0134215228..94ac44fc84 100644 --- a/src/aliceVision/panorama/laplacianCompositer.hpp +++ b/src/aliceVision/panorama/laplacianCompositer.hpp @@ -42,7 +42,8 @@ class LaplacianCompositer : public Compositer int gaussianFilterSize = 1 + 2 * _gaussianFilterRadius; size_t optimal_scale = size_t(floor(std::log2(double(minsize) / gaussianFilterSize))); - return optimal_scale; + + return (optimal_scale - 1/*Security*/); } virtual int getBorderSize() const @@ -53,7 +54,7 @@ class LaplacianCompositer : public Compositer virtual bool append(const aliceVision::image::Image& color, const aliceVision::image::Image& inputMask, const aliceVision::image::Image& inputWeights, - int offset_x, int offset_y) + int offsetX, int offsetY) { size_t optimalScale = getOptimalScale(color.Width(), color.Height()); size_t optimalLevelsCount = optimalScale + 1; @@ -69,23 +70,26 @@ class LaplacianCompositer : public Compositer if(optimalLevelsCount > _bands) { _bands = optimalLevelsCount; - _pyramidPanorama.augment(_cacheManager, _bands); + if (!_pyramidPanorama.augment(_cacheManager, _bands)) + { + return false; + } } // Make sure input is compatible with pyramid processing - size_t new_offset_x, new_offset_y; - aliceVision::image::Image color_pot; - aliceVision::image::Image mask_pot; - aliceVision::image::Image weights_pot; + size_t newOffsetX, newOffsetY; + aliceVision::image::Image colorPot; + aliceVision::image::Image maskPot; + aliceVision::image::Image weightsPot; - makeImagePyramidCompatible(color_pot, new_offset_x, new_offset_y, color, offset_x, offset_y, _bands); - makeImagePyramidCompatible(mask_pot, new_offset_x, new_offset_y, inputMask, offset_x, offset_y, _bands); - makeImagePyramidCompatible(weights_pot, new_offset_x, new_offset_y, inputWeights, offset_x, offset_y, _bands); + makeImagePyramidCompatible(colorPot, newOffsetX, newOffsetY, color, offsetX, offsetY, _bands); + makeImagePyramidCompatible(maskPot, newOffsetX, newOffsetY, inputMask, offsetX, offsetY, _bands); + makeImagePyramidCompatible(weightsPot, newOffsetX, newOffsetY, inputWeights, offsetX, offsetY, _bands); // Fill Color images masked parts with fake but coherent info aliceVision::image::Image feathered; - if (!feathering(feathered, color_pot, mask_pot)) + if (!feathering(feathered, colorPot, maskPot)) { return false; } @@ -101,7 +105,24 @@ class LaplacianCompositer : public Compositer } } - if (!_pyramidPanorama.apply(feathered, mask_pot, weights_pot, new_offset_x, new_offset_y)) + /* Convert mask to alpha layer */ + image::Image maskFloat(maskPot.Width(), maskPot.Height()); + for(int i = 0; i < maskPot.Height(); i++) + { + for(int j = 0; j < maskPot.Width(); j++) + { + if(maskPot(i, j)) + { + maskFloat(i, j) = 1.0f; + } + else + { + maskFloat(i, j) = 0.0f; + } + } + } + + if (!_pyramidPanorama.apply(feathered, maskFloat, weightsPot, 0, newOffsetX, newOffsetY)) { return false; } @@ -134,7 +155,7 @@ class LaplacianCompositer : public Compositer } protected: - const float _gaussianFilterRadius = 2.0f; + const int _gaussianFilterRadius = 2; LaplacianPyramid _pyramidPanorama; size_t _bands; }; diff --git a/src/aliceVision/panorama/laplacianPyramid.cpp b/src/aliceVision/panorama/laplacianPyramid.cpp index 0ef9878c2b..32e0b40f50 100644 --- a/src/aliceVision/panorama/laplacianPyramid.cpp +++ b/src/aliceVision/panorama/laplacianPyramid.cpp @@ -80,76 +80,6 @@ bool LaplacianPyramid::augment(image::TileCacheManager::shared_ptr & cacheManage int oldMaxLevels = _levels.size(); _maxLevels = newMaxLevels; - //Get content of last level of pyramid - CachedImage largerColor; - if(!largerColor.createImage(cacheManager, _levels[oldMaxLevels - 1].getWidth(), _levels[oldMaxLevels - 1].getHeight())) - { - return false; - } - - if (!largerColor.deepCopy(_levels[oldMaxLevels - 1])) - { - return false; - } - - //Get weights of last level of pyramid - CachedImage largerWeight; - if(!largerWeight.createImage(cacheManager, _weights[oldMaxLevels - 1].getWidth(), _weights[oldMaxLevels - 1].getHeight())) - { - return false; - } - - if (!largerWeight.deepCopy(_weights[oldMaxLevels - 1])) - { - return false; - } - - // Last level was multiplied by the weight. - // Remove this factor - largerColor.perPixelOperation(largerWeight, - [](const image::RGBfColor & c, const float & w) -> image::RGBfColor - { - if (w < 1e-6) - { - return image::RGBfColor(0.0f, 0.0f, 0.0f); - } - - image::RGBfColor r; - - r.r() = c.r() / w; - r.g() = c.g() / w; - r.b() = c.b() / w; - - return r; - } - ); - - // Create a mask - CachedImage largerMask; - if(!largerMask.createImage(cacheManager, largerWeight.getWidth(), largerWeight.getHeight())) - { - return false; - } - - // Build the mask - largerMask.perPixelOperation(largerWeight, - [](const unsigned char & c, const float & w) -> unsigned char - { - if (w < 1e-6) - { - return 0; - } - - return 255; - } - ); - - - // Put colors in masked areas - if (!feathering(largerColor, largerMask)) - { - return false; - } // Augment the number of levels in the pyramid for (int level = oldMaxLevels; level < newMaxLevels; level++) @@ -189,225 +119,41 @@ bool LaplacianPyramid::augment(image::TileCacheManager::shared_ptr & cacheManage _weights.push_back(pyramidWeights); } - const int processingSize = 512; - const size_t borderSize = 5; - CachedImage currentImage = largerColor; - CachedImage currentWeights = largerWeight; - - for (int level = oldMaxLevels - 1; level < _maxLevels - 1; level++) + std::vector localInfos = _inputInfos; + _inputInfos.clear(); + + for (InputInfo & iinfo : localInfos) { - int width = currentImage.getWidth(); - int height = currentImage.getHeight(); - int nextWidth = int(ceil(float(width) / 2.0f)); - int nextHeight = int(ceil(float(height) / 2.0f)); - - - // Create buffer for next level - CachedImage nextImage; - CachedImage nextWeights; - - if(!nextImage.createImage(cacheManager, nextWidth, nextHeight)) - { - return false; - } - - if(!nextWeights.createImage(cacheManager, nextWidth, nextHeight)) + if (!apply(iinfo.color, iinfo.mask, iinfo.weights, oldMaxLevels - 1, iinfo.offsetX, iinfo.offsetY)) { return false; } - - nextImage.fill(image::RGBfColor(0.0f)); - nextWeights.fill(0.0f); - - - //Process image rectangle by rectangle - for (int y = 0; y < nextImage.getHeight(); y += processingSize) - { - for (int x = 0; x < nextImage.getWidth(); x += processingSize) - { - //Compute the initial bouding box for this rectangle - BoundingBox nextBbox; - nextBbox.left = x; - nextBbox.top = y; - nextBbox.width = processingSize; - nextBbox.height = processingSize; - nextBbox.clampLeft(); - nextBbox.clampTop(); - nextBbox.clampRight(nextImage.getWidth() - 1); - nextBbox.clampBottom(nextImage.getHeight() - 1); - - - // Dilate the bounding box to account for filtering - BoundingBox dilatedNextBbox = nextBbox.dilate(borderSize); - dilatedNextBbox.clampTop(); - dilatedNextBbox.clampBottom(nextImage.getHeight() - 1); - - //If the box has a negative left border, - //it is equivalent (with the loop, to shifting at the end) - int leftBorder = nextBbox.left - dilatedNextBbox.left; - int topBorder = nextBbox.top - dilatedNextBbox.top; - if (dilatedNextBbox.left < 0) - { - dilatedNextBbox.left += nextImage.getWidth(); - } - - //Same box in the current level - BoundingBox currentBbox = nextBbox.doubleSize(); - currentBbox.clampLeft(); - currentBbox.clampTop(); - currentBbox.clampRight(currentImage.getWidth() - 1); - currentBbox.clampBottom(currentImage.getHeight() - 1); - - //Dilated box in the current level - BoundingBox dilatedCurrentBbox = dilatedNextBbox.doubleSize(); - currentBbox.clampTop(); - currentBbox.clampBottom(currentImage.getHeight() - 1); - - //Extract the image with borders in the current level - aliceVision::image::Image extractedColor(dilatedCurrentBbox.width, dilatedCurrentBbox.height); - if (!loopyCachedImageExtract(extractedColor, currentImage, dilatedCurrentBbox)) - { - return false; - } - - //Extract the weights with borders in the current level - aliceVision::image::Image extractedWeight(dilatedCurrentBbox.width, dilatedCurrentBbox.height); - if (!loopyCachedImageExtract(extractedWeight, currentWeights, dilatedCurrentBbox)) - { - return false; - } - - //Filter current image - aliceVision::image::Image buf(dilatedCurrentBbox.width, dilatedCurrentBbox.height); - aliceVision::image::Image bufw(dilatedCurrentBbox.width, dilatedCurrentBbox.height); - convolveGaussian5x5(buf, extractedColor); - convolveGaussian5x5(bufw, extractedWeight); - - //Downscale current image to next level - aliceVision::image::Image colorDownscaled(dilatedNextBbox.width, dilatedNextBbox.height); - aliceVision::image::Image weightDownscaled(dilatedNextBbox.width, dilatedNextBbox.height); - downscale(colorDownscaled, buf); - downscale(weightDownscaled, bufw); - - BoundingBox saveBoundingBox; - saveBoundingBox.left = leftBorder; - saveBoundingBox.top = topBorder; - saveBoundingBox.width = nextBbox.width; - saveBoundingBox.height = nextBbox.height; - - if (!loopyCachedImageAssign(nextImage, colorDownscaled, nextBbox, saveBoundingBox)) { - return false; - } - - if (!loopyCachedImageAssign(nextWeights, weightDownscaled, nextBbox, saveBoundingBox)) { - return false; - } - - - - /* Compute difference */ - aliceVision::image::Image buf2(dilatedCurrentBbox.width, dilatedCurrentBbox.height); - upscale(buf, colorDownscaled); - convolveGaussian5x5(buf2, buf); - - for (int i = 0; i < buf2.Height(); i++) { - for (int j = 0; j < buf2.Width(); j++) { - - buf2(i,j) *= 4.0f; - } - } - - substract(extractedColor, extractedColor, buf2); - - for (int i = 0; i < extractedColor.Height(); i++) - { - for (int j = 0; j < extractedColor.Width(); j++) - { - extractedColor(i, j).r() = extractedColor(i, j).r() * extractedWeight(i, j); - extractedColor(i, j).g() = extractedColor(i, j).r() * extractedWeight(i, j); - extractedColor(i, j).b() = extractedColor(i, j).b() * extractedWeight(i, j); - } - } - - saveBoundingBox.left = leftBorder * 2; - saveBoundingBox.top = topBorder * 2; - saveBoundingBox.width = currentBbox.width; - saveBoundingBox.height = currentBbox.height; - - if (!loopyCachedImageAssign(_levels[level], extractedColor, currentBbox, saveBoundingBox)) { - return false; - } - - if (!loopyCachedImageAssign(_weights[level], extractedWeight, currentBbox, saveBoundingBox)) { - return false; - } - } - } - - currentImage = nextImage; - currentWeights = nextWeights; } - currentImage.perPixelOperation(currentWeights, - [](const image::RGBfColor & c, const float & w) -> image::RGBfColor - { - image::RGBfColor r; - - r.r() = c.r() * w; - r.g() = c.g() * w; - r.b() = c.b() * w; - - return r; - } - ); - - - _levels[_levels.size() - 1].deepCopy(currentImage); - _weights[_weights.size() - 1].deepCopy(currentWeights); - - return true; } bool LaplacianPyramid::apply(const aliceVision::image::Image& source, - const aliceVision::image::Image& mask, + const aliceVision::image::Image& mask, const aliceVision::image::Image& weights, - size_t offsetX, size_t offsetY) + size_t initialLevel, size_t offsetX, size_t offsetY) { //We assume the input source has been feathered //and resized to be a simili power of 2 for the needed scales. int width = source.Width(); int height = source.Height(); - /* Convert mask to alpha layer */ - image::Image mask_float(width, height); - for(int i = 0; i < height; i++) - { - for(int j = 0; j < width; j++) - { - if(mask(i, j)) - { - mask_float(i, j) = 1.0f; - } - else - { - mask_float(i, j) = 0.0f; - } - } - } - - image::Image currentColor = source; image::Image nextColor; image::Image currentWeights = weights; image::Image nextWeights; - image::Image currentMask = mask_float; + image::Image currentMask = mask; image::Image nextMask; - for(int l = 0; l < _levels.size() - 1; l++) + for(int l = initialLevel; l < _levels.size() - 1; l++) { CachedImage & img = _levels[l]; @@ -611,10 +357,18 @@ bool LaplacianPyramid::apply(const aliceVision::image::Image& offsetY = offsetY / 2; } - if (!merge(currentColor, currentWeights, _levels.size() - 1, offsetX, offsetY)) + InputInfo iinfo; + iinfo.offsetX = offsetX; + iinfo.offsetY = offsetY; + iinfo.color = currentColor; + iinfo.mask = currentMask; + iinfo.weights = currentWeights; + _inputInfos.push_back(iinfo); + + /*if (!merge(currentColor, currentWeights, _levels.size() - 1, offsetX, offsetY)) { return false; - } + }*/ return true; } @@ -675,6 +429,14 @@ bool LaplacianPyramid::merge(const aliceVision::image::Image& bool LaplacianPyramid::rebuild(CachedImage& output) { + for (InputInfo & iinfo : _inputInfos) + { + if (!merge(iinfo.color, iinfo.weights, _levels.size() - 1, iinfo.offsetX, iinfo.offsetY)) + { + return false; + } + } + // We first want to compute the final pixels mean for(int l = 0; l < _levels.size(); l++) { diff --git a/src/aliceVision/panorama/laplacianPyramid.hpp b/src/aliceVision/panorama/laplacianPyramid.hpp index 46b7ffcc5e..7185eb6ba0 100644 --- a/src/aliceVision/panorama/laplacianPyramid.hpp +++ b/src/aliceVision/panorama/laplacianPyramid.hpp @@ -9,6 +9,16 @@ namespace aliceVision class LaplacianPyramid { +public: + struct InputInfo + { + aliceVision::image::Image color; + aliceVision::image::Image mask; + aliceVision::image::Image weights; + int offsetX; + int offsetY; + }; + public: LaplacianPyramid(size_t base_width, size_t base_height, size_t max_levels); @@ -17,8 +27,8 @@ class LaplacianPyramid bool augment(image::TileCacheManager::shared_ptr & cacheManager, size_t new_max_levels); bool apply(const aliceVision::image::Image& source, - const aliceVision::image::Image& mask, const aliceVision::image::Image& weights, - size_t offset_x, size_t offset_y); + const aliceVision::image::Image& mask, const aliceVision::image::Image& weights, + size_t initialLevel, size_t offset_x, size_t offset_y); bool merge(const aliceVision::image::Image& oimg, const aliceVision::image::Image& oweight, size_t level, size_t offset_x, size_t offset_y); @@ -33,6 +43,7 @@ class LaplacianPyramid std::vector> _levels; std::vector> _weights; std::vector _realWidths; + std::vector _inputInfos; }; } // namespace aliceVision \ No newline at end of file diff --git a/src/software/pipeline/main_panoramaCompositing.cpp b/src/software/pipeline/main_panoramaCompositing.cpp index 494a478bc9..f7a1d1e8b9 100644 --- a/src/software/pipeline/main_panoramaCompositing.cpp +++ b/src/software/pipeline/main_panoramaCompositing.cpp @@ -390,7 +390,7 @@ int aliceVision_main(int argc, char** argv) ALICEVISION_LOG_INFO(viewOrderedByScale.size() << " views to process"); - BoundingBoxPanoramaMap map(panoramaSize.first, panoramaSize.second); + /*BoundingBoxPanoramaMap map(panoramaSize.first, panoramaSize.second); if (!buildMap(map, viewOrderedByScale, warpingFolder, compositer)) { ALICEVISION_LOG_ERROR("Error computing map"); @@ -402,7 +402,7 @@ int aliceVision_main(int argc, char** argv) { ALICEVISION_LOG_ERROR("Error computing best elements"); return EXIT_SUCCESS; - } + }*/ CachedImage labels; if (useSeams) From 01b1c7e23e3650eb6ac08c5cb262336e301893a3 Mon Sep 17 00:00:00 2001 From: Fabien Servant Date: Mon, 2 Nov 2020 09:48:07 +0100 Subject: [PATCH 28/79] [panorama] Some minor changes --- src/aliceVision/panorama/imageOps.hpp | 49 ++++++++++--------- .../panorama/laplacianCompositer.hpp | 7 +-- src/aliceVision/panorama/seams.cpp | 4 +- 3 files changed, 31 insertions(+), 29 deletions(-) diff --git a/src/aliceVision/panorama/imageOps.hpp b/src/aliceVision/panorama/imageOps.hpp index f1f90b9a08..37b7b0d8ae 100644 --- a/src/aliceVision/panorama/imageOps.hpp +++ b/src/aliceVision/panorama/imageOps.hpp @@ -279,9 +279,10 @@ bool loopyCachedImageExtract(aliceVision::image::Image & output, CachedImage< template bool makeImagePyramidCompatible(image::Image& output, - size_t& out_offset_x, size_t& out_offset_y, + size_t& outOffsetX, size_t& outOffsetY, const image::Image& input, - size_t offset_x, size_t offset_y, + size_t offsetX, size_t offsetY, + size_t borderSize, size_t num_levels) { @@ -290,49 +291,49 @@ bool makeImagePyramidCompatible(image::Image& output, return false; } - double max_scale = 1.0 / pow(2.0, num_levels - 1); + double maxScale = 1.0 / pow(2.0, num_levels - 1); - double low_offset_x = double(offset_x) * max_scale; - double low_offset_y = double(offset_y) * max_scale; + double lowOffsetX = double(offsetX) * maxScale; + double lowOffsetY = double(offsetY) * maxScale; /*Make sure offset is integer even at the lowest level*/ - double corrected_low_offset_x = floor(low_offset_x); - double corrected_low_offset_y = floor(low_offset_y); + double correctedLowOffsetX = floor(lowOffsetX); + double correctedLowOffsetY = floor(lowOffsetY); /*Add some borders on the top and left to make sure mask can be smoothed*/ - corrected_low_offset_x = std::max(0.0, corrected_low_offset_x - 3.0); - corrected_low_offset_y = std::max(0.0, corrected_low_offset_y - 3.0); + correctedLowOffsetX = std::max(0.0, correctedLowOffsetX - double(borderSize)); + correctedLowOffsetY = std::max(0.0, correctedLowOffsetY - double(borderSize)); /*Compute offset at largest level*/ - out_offset_x = size_t(corrected_low_offset_x / max_scale); - out_offset_y = size_t(corrected_low_offset_y / max_scale); + outOffsetX = size_t(correctedLowOffsetX / maxScale); + outOffsetY = size_t(correctedLowOffsetY / maxScale); /*Compute difference*/ - double doffset_x = double(offset_x) - double(out_offset_x); - double doffset_y = double(offset_y) - double(out_offset_y); + double doffsetX = double(offsetX) - double(outOffsetX); + double doffsetY = double(offsetY) - double(outOffsetY); /* update size with border update */ - double large_width = double(input.Width()) + doffset_x; - double large_height = double(input.Height()) + doffset_y; + double large_width = double(input.Width()) + doffsetX; + double large_height = double(input.Height()) + doffsetY; /* compute size at largest scale */ - double low_width = large_width * max_scale; - double low_height = large_height * max_scale; + double low_width = large_width * maxScale; + double low_height = large_height * maxScale; /*Make sure width is integer event at the lowest level*/ - double corrected_low_width = ceil(low_width); - double corrected_low_height = ceil(low_height); + double correctedLowWidth = ceil(low_width); + double correctedLowHeight = ceil(low_height); /*Add some borders on the right and bottom to make sure mask can be smoothed*/ - corrected_low_width = corrected_low_width + 3; - corrected_low_height = corrected_low_height + 3; + correctedLowWidth = correctedLowWidth + double(borderSize); + correctedLowHeight = correctedLowHeight + double(borderSize); /*Compute size at largest level*/ - size_t width = size_t(corrected_low_width / max_scale); - size_t height = size_t(corrected_low_height / max_scale); + size_t width = size_t(correctedLowWidth / maxScale); + size_t height = size_t(correctedLowHeight / maxScale); output = image::Image(width, height, true, T(0.0f)); - output.block(doffset_y, doffset_x, input.Height(), input.Width()) = input; + output.block(doffsetY, doffsetX, input.Height(), input.Width()) = input; return true; } diff --git a/src/aliceVision/panorama/laplacianCompositer.hpp b/src/aliceVision/panorama/laplacianCompositer.hpp index 94ac44fc84..5d693c9159 100644 --- a/src/aliceVision/panorama/laplacianCompositer.hpp +++ b/src/aliceVision/panorama/laplacianCompositer.hpp @@ -77,14 +77,15 @@ class LaplacianCompositer : public Compositer } // Make sure input is compatible with pyramid processing + // See comments inside function for details size_t newOffsetX, newOffsetY; aliceVision::image::Image colorPot; aliceVision::image::Image maskPot; aliceVision::image::Image weightsPot; - makeImagePyramidCompatible(colorPot, newOffsetX, newOffsetY, color, offsetX, offsetY, _bands); - makeImagePyramidCompatible(maskPot, newOffsetX, newOffsetY, inputMask, offsetX, offsetY, _bands); - makeImagePyramidCompatible(weightsPot, newOffsetX, newOffsetY, inputWeights, offsetX, offsetY, _bands); + makeImagePyramidCompatible(colorPot, newOffsetX, newOffsetY, color, offsetX, offsetY, getBorderSize(), _bands); + makeImagePyramidCompatible(maskPot, newOffsetX, newOffsetY, inputMask, offsetX, offsetY, getBorderSize(), _bands); + makeImagePyramidCompatible(weightsPot, newOffsetX, newOffsetY, inputWeights, offsetX, offsetY, getBorderSize(), _bands); // Fill Color images masked parts with fake but coherent info diff --git a/src/aliceVision/panorama/seams.cpp b/src/aliceVision/panorama/seams.cpp index ced5c0dbee..d9e6b0414c 100644 --- a/src/aliceVision/panorama/seams.cpp +++ b/src/aliceVision/panorama/seams.cpp @@ -430,8 +430,8 @@ bool HierarchicalGraphcutSeams::append(const aliceVision::image::Image potImage; aliceVision::image::Image potMask; - makeImagePyramidCompatible(potImage, newOffsetX, newOffsetY, input, offsetX, offsetY, _countLevels); - makeImagePyramidCompatible(potMask, newOffsetX, newOffsetY, inputMask, offsetX, offsetY, _countLevels); + makeImagePyramidCompatible(potImage, newOffsetX, newOffsetY, input, offsetX, offsetY, 2, _countLevels); + makeImagePyramidCompatible(potMask, newOffsetX, newOffsetY, inputMask, offsetX, offsetY, 2, _countLevels); // Fill Color images masked parts with fake but coherent info aliceVision::image::Image feathered; From 915930fa54fd63958157010065cb4ef8e1476d08 Mon Sep 17 00:00:00 2001 From: Fabien Servant Date: Thu, 5 Nov 2020 08:21:52 +0100 Subject: [PATCH 29/79] [panorama] add maxPanoramaWidth --- .../pipeline/main_panoramaWarping.cpp | 23 ++++++++++++------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/src/software/pipeline/main_panoramaWarping.cpp b/src/software/pipeline/main_panoramaWarping.cpp index 0e74fd3910..06722de793 100644 --- a/src/software/pipeline/main_panoramaWarping.cpp +++ b/src/software/pipeline/main_panoramaWarping.cpp @@ -101,6 +101,7 @@ int aliceVision_main(int argc, char** argv) std::pair panoramaSize = {0, 0}; int percentUpscale = 50; int tileSize = 256; + int maxPanoramaWidth = 0; image::EStorageDataType storageDataType = image::EStorageDataType::Float; @@ -120,14 +121,14 @@ int aliceVision_main(int argc, char** argv) // Description of optional parameters po::options_description optionalParams("Optional parameters"); - optionalParams.add_options()( - "panoramaWidth,w", po::value(&panoramaSize.first)->default_value(panoramaSize.first), - "Panorama Width in pixels.")("percentUpscale", po::value(&percentUpscale)->default_value(percentUpscale), - "Percentage of upscaled pixels.")( - "storageDataType", po::value(&storageDataType)->default_value(storageDataType), - ("Storage data type: " + image::EStorageDataType_informations()).c_str())( - "rangeStart", po::value(&rangeStart)->default_value(rangeStart), - "Range image index start.")("rangeSize", po::value(&rangeSize)->default_value(rangeSize), "Range size."); + optionalParams.add_options() + ("panoramaWidth,w", po::value(&panoramaSize.first)->default_value(panoramaSize.first), "Panorama Width in pixels.") + ("maxPanoramaWidth", po::value(&maxPanoramaWidth)->default_value(maxPanoramaWidth), "Max Panorama Width in pixels.") + ("percentUpscale", po::value(&percentUpscale)->default_value(percentUpscale), "Percentage of upscaled pixels.") + ("storageDataType", po::value(&storageDataType)->default_value(storageDataType), + ("Storage data type: " + image::EStorageDataType_informations()).c_str()) + ("rangeStart", po::value(&rangeStart)->default_value(rangeStart), "Range image index start.") + ("rangeSize", po::value(&rangeSize)->default_value(rangeSize), "Range size."); allParams.add(optionalParams); // Setup log level given command line @@ -240,6 +241,12 @@ int aliceVision_main(int argc, char** argv) ALICEVISION_LOG_INFO("Impossible to compute an optimal panorama size"); return EXIT_FAILURE; } + + if (maxPanoramaWidth != 0 && panoramaSize.first > maxPanoramaWidth) + { + ALICEVISION_LOG_INFO("The optimal size of the panorama exceeds the maximum size (estimated width: " << panoramaSize.first << ", max width: " << maxPanoramaWidth << ")."); + panoramaSize.first = maxPanoramaWidth; + } } // Make sure panorama size has required size properties From 138500c9ca2be77a654e0b3b58c83b3af0296ce3 Mon Sep 17 00:00:00 2001 From: Fabien Date: Fri, 6 Nov 2020 17:58:33 +0100 Subject: [PATCH 30/79] [panorama] forgot a debug comment in sfm --- src/aliceVision/sfm/BundleAdjustmentSymbolicCeres.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aliceVision/sfm/BundleAdjustmentSymbolicCeres.cpp b/src/aliceVision/sfm/BundleAdjustmentSymbolicCeres.cpp index 753868708d..c358b4bf84 100644 --- a/src/aliceVision/sfm/BundleAdjustmentSymbolicCeres.cpp +++ b/src/aliceVision/sfm/BundleAdjustmentSymbolicCeres.cpp @@ -276,7 +276,7 @@ void BundleAdjustmentSymbolicCeres::setSolverOptions(ceres::Solver::Options& sol solverOptions.sparse_linear_algebra_library_type = _ceresOptions.sparseLinearAlgebraLibraryType; solverOptions.minimizer_progress_to_stdout = _ceresOptions.verbose; solverOptions.logging_type = ceres::SILENT; - solverOptions.num_threads = 1;_ceresOptions.nbThreads; + solverOptions.num_threads = _ceresOptions.nbThreads; #if CERES_VERSION_MAJOR < 2 solverOptions.num_linear_solver_threads = _ceresOptions.nbThreads; From 024f8835b577c85bb743322de74675fa7e65c243 Mon Sep 17 00:00:00 2001 From: Fabien Date: Fri, 6 Nov 2020 17:58:50 +0100 Subject: [PATCH 31/79] [panorama] cache path --- src/software/pipeline/main_panoramaCompositing.cpp | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/software/pipeline/main_panoramaCompositing.cpp b/src/software/pipeline/main_panoramaCompositing.cpp index f7a1d1e8b9..7c0ebe48f4 100644 --- a/src/software/pipeline/main_panoramaCompositing.cpp +++ b/src/software/pipeline/main_panoramaCompositing.cpp @@ -184,6 +184,7 @@ int aliceVision_main(int argc, char** argv) std::string sfmDataFilepath; std::string warpingFolder; std::string outputPanorama; + std::string temporaryCachePath; std::string compositerType = "multiband"; std::string overlayType = "none"; @@ -204,9 +205,11 @@ int aliceVision_main(int argc, char** argv) // Description of mandatory parameters po::options_description requiredParams("Required parameters"); - requiredParams.add_options()("input,i", po::value(&sfmDataFilepath)->required(), "Input sfmData.")( - "warpingFolder,w", po::value(&warpingFolder)->required(), "Folder with warped images.")( - "output,o", po::value(&outputPanorama)->required(), "Path of the output panorama."); + requiredParams.add_options() + ("input,i", po::value(&sfmDataFilepath)->required(), "Input sfmData.") + ("warpingFolder,w", po::value(&warpingFolder)->required(), "Folder with warped images.") + ("output,o", po::value(&outputPanorama)->required(), "Path of the output panorama.") + ("cacheFolder,f", po::value(&temporaryCachePath)->required(), "Path of the temporary cache."); allParams.add(requiredParams); // Description of optional parameters @@ -214,7 +217,7 @@ int aliceVision_main(int argc, char** argv) optionalParams.add_options()("compositerType,c", po::value(&compositerType)->required(), "Compositer Type [replace, alpha, multiband].")( "overlayType,c", po::value(&overlayType)->required(), "Overlay Type [none, borders, seams, all].")( - "useGraphCut,c", po::value(&useGraphCut)->default_value(useGraphCut), + "useGraphCut,g", po::value(&useGraphCut)->default_value(useGraphCut), "Do we use graphcut for ghost removal ?")( "storageDataType", po::value(&storageDataType)->default_value(storageDataType), ("Storage data type: " + image::EStorageDataType_informations()).c_str()); @@ -296,8 +299,7 @@ int aliceVision_main(int argc, char** argv) } // Create a cache manager - const std::string outputPath = boost::filesystem::path(outputPanorama).parent_path().string(); - image::TileCacheManager::shared_ptr cacheManager = image::TileCacheManager::create(outputPath, 256, 256, 65536); + image::TileCacheManager::shared_ptr cacheManager = image::TileCacheManager::create(temporaryCachePath, 256, 256, 65536); if(!cacheManager) { ALICEVISION_LOG_ERROR("Error creating the cache manager"); From 2aba949ba6135903a05f987e8e55bde7dadf3cbb Mon Sep 17 00:00:00 2001 From: Fabien Date: Fri, 13 Nov 2020 17:44:49 +0100 Subject: [PATCH 32/79] [panorama] change upscale --- src/aliceVision/panorama/imageOps.hpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/aliceVision/panorama/imageOps.hpp b/src/aliceVision/panorama/imageOps.hpp index 37b7b0d8ae..c696fc7421 100644 --- a/src/aliceVision/panorama/imageOps.hpp +++ b/src/aliceVision/panorama/imageOps.hpp @@ -43,10 +43,10 @@ bool upscale(aliceVision::image::Image& outputColor, const aliceVision::image { int dj = j * 2; - outputColor(di, dj) = T(); + outputColor(di, dj) = inputColor(i, j); outputColor(di, dj + 1) = T(); outputColor(di + 1, dj) = T(); - outputColor(di + 1, dj + 1) = inputColor(i, j); + outputColor(di + 1, dj + 1) = T(); } } @@ -56,7 +56,7 @@ bool upscale(aliceVision::image::Image& outputColor, const aliceVision::image int di = i * 2; int dj = j * 2; - outputColor(di, dj) = T(); + outputColor(di, dj) = inputColor(i, j); if (dj < dwidth - 1) { @@ -69,7 +69,7 @@ bool upscale(aliceVision::image::Image& outputColor, const aliceVision::image if (dj < dwidth - 1) { - outputColor(di + 1, dj + 1) = inputColor(i, j); + outputColor(di + 1, dj + 1) = T(); } } } @@ -80,7 +80,7 @@ bool upscale(aliceVision::image::Image& outputColor, const aliceVision::image int di = i * 2; int dj = j * 2; - outputColor(di, dj) = T(); + outputColor(di, dj) = inputColor(i, j); if (dj < dwidth - 1) { @@ -93,7 +93,7 @@ bool upscale(aliceVision::image::Image& outputColor, const aliceVision::image if (dj < dwidth - 1) { - outputColor(di + 1, dj + 1) = inputColor(i, j); + outputColor(di + 1, dj + 1) = T(); } } } From a7843ff423ba64a00b65a46945d28bea407c00c8 Mon Sep 17 00:00:00 2001 From: Fabien Date: Wed, 18 Nov 2020 10:12:44 +0100 Subject: [PATCH 33/79] [panorama] pyramid management for graphcut --- src/aliceVision/panorama/graphcut.hpp | 299 +++++++++++++++++--------- src/aliceVision/panorama/seams.cpp | 31 ++- 2 files changed, 208 insertions(+), 122 deletions(-) diff --git a/src/aliceVision/panorama/graphcut.hpp b/src/aliceVision/panorama/graphcut.hpp index cb00803496..2f0e6ff7fc 100644 --- a/src/aliceVision/panorama/graphcut.hpp +++ b/src/aliceVision/panorama/graphcut.hpp @@ -145,7 +145,7 @@ class GraphcutSeams CachedImage mask; }; - using PixelInfo = std::map; + using PixelInfo = std::unordered_map; public: @@ -211,65 +211,8 @@ class GraphcutSeams _maximal_distance_change = dist; } - bool processInput(double & newCost, InputData & input) + bool createInputOverlappingObservations(image::Image & graphCutInput, const BoundingBox & interestBbox) { - //Get bounding box of input in panoram - //Dilate to have some pixels outside of the input - BoundingBox localBbox = input.rect.dilate(3); - localBbox.clampLeft(); - localBbox.clampTop(); - localBbox.clampBottom(_labels.getHeight() - 1); - - //Output must keep a margin also - BoundingBox outputBbox = input.rect; - outputBbox.left = input.rect.left - localBbox.left; - outputBbox.top = input.rect.top - localBbox.top; - - //Extract labels for the ROI - image::Image localLabels(localBbox.width, localBbox.height); - if (!loopyCachedImageExtract(localLabels, _labels, localBbox)) - { - return false; - } - - // Compute distance map to borders of the input seams - image::Image distanceMap(localLabels.Width(), localLabels.Height()); - { - image::Image binarizedWorld(localLabels.Width(), localLabels.Height()); - - for(int y = 0; y < localLabels.Height(); y++) - { - for(int x = 0; x < localLabels.Width(); x++) - { - IndexT label = localLabels(y, x); - - if(label == input.id) - { - binarizedWorld(y, x) = 1; - } - else - { - binarizedWorld(y, x) = 0; - } - } - } - - image::Image seams(localLabels.Width(), localLabels.Height()); - if(!computeSeamsMap(seams, binarizedWorld)) - { - return false; - } - - if(!computeDistanceMap(distanceMap, seams)) - { - return false; - } - } - - - //Build the input - image::Image graphCutInput(localBbox.width, localBbox.height, true); - for (auto & otherInput : _inputs) { BoundingBox otherBbox = otherInput.second.rect; @@ -278,13 +221,14 @@ class GraphcutSeams BoundingBox otherBboxLoopRight = otherInput.second.rect; otherBboxLoopRight.left = otherBbox.left + _outputWidth; + BoundingBox otherInputBbox = otherBbox; otherInputBbox.left = 0; otherInputBbox.top = 0; - BoundingBox intersection = localBbox.intersectionWith(otherBbox); - BoundingBox intersectionLoop = localBbox.intersectionWith(otherBboxLoop); - BoundingBox intersectionLoopRight = localBbox.intersectionWith(otherBboxLoopRight); + BoundingBox intersection = interestBbox.intersectionWith(otherBbox); + BoundingBox intersectionLoop = interestBbox.intersectionWith(otherBboxLoop); + BoundingBox intersectionLoopRight = interestBbox.intersectionWith(otherBboxLoopRight); if (intersection.isEmpty() && intersectionLoop.isEmpty() && intersectionLoopRight.isEmpty()) { continue; @@ -302,7 +246,6 @@ class GraphcutSeams return false; } - if (!intersection.isEmpty()) { BoundingBox interestOther = intersection; @@ -310,8 +253,8 @@ class GraphcutSeams interestOther.top -= otherBbox.top; BoundingBox interestThis = intersection; - interestThis.left -= localBbox.left; - interestThis.top -= localBbox.top; + interestThis.left -= interestBbox.left; + interestThis.top -= interestBbox.top; for (int y = 0; y < intersection.height; y++) { @@ -323,17 +266,12 @@ class GraphcutSeams int x_other = interestOther.left + x; int x_current = interestThis.left + x; + if (!otherMask(y_other, x_other)) { continue; } - - float dist = sqrt(float(distanceMap(y_current, x_current))); - if (dist > _maximal_distance_change) - { - continue; - } - + PixelInfo & pix = graphCutInput(y_current, x_current); pix[otherInput.first] = otherColor(y_other, x_other); } @@ -347,8 +285,8 @@ class GraphcutSeams interestOther.top -= otherBboxLoop.top; BoundingBox interestThis = intersectionLoop; - interestThis.left -= localBbox.left; - interestThis.top -= localBbox.top; + interestThis.left -= interestBbox.left; + interestThis.top -= interestBbox.top; for (int y = 0; y < intersectionLoop.height; y++) @@ -366,12 +304,6 @@ class GraphcutSeams continue; } - float dist = sqrt(float(distanceMap(y_current, x_current))); - if (dist > _maximal_distance_change) - { - continue; - } - PixelInfo & pix = graphCutInput(y_current, x_current); pix[otherInput.first] = otherColor(y_other, x_other); } @@ -385,8 +317,8 @@ class GraphcutSeams interestOther.top -= otherBboxLoopRight.top; BoundingBox interestThis = intersectionLoopRight; - interestThis.left -= localBbox.left; - interestThis.top -= localBbox.top; + interestThis.left -= interestBbox.left; + interestThis.top -= interestBbox.top; for (int y = 0; y < intersectionLoopRight.height; y++) @@ -404,12 +336,6 @@ class GraphcutSeams continue; } - float dist = sqrt(float(distanceMap(y_current, x_current))); - if (dist > _maximal_distance_change) - { - continue; - } - PixelInfo & pix = graphCutInput(y_current, x_current); pix[otherInput.first] = otherColor(y_other, x_other); } @@ -417,40 +343,192 @@ class GraphcutSeams } } + return true; + } + + bool fixUpscaling(image::Image & labels, const image::Image & graphCutInput) + { //Because of upscaling, some labels may be incorrect //Some pixels may be affected to labels they don't see. for (int y = 0; y < graphCutInput.Height(); y++) { for (int x = 0; x < graphCutInput.Width(); x++) { - IndexT label = localLabels(y, x); + IndexT label = labels(y, x); if (label == UndefinedIndexT) { continue; } - float dist = sqrt(float(distanceMap(y, x))); - if (dist > _maximal_distance_change) + // If there is no input for this pixel, we can't do anything + const PixelInfo & pix = graphCutInput(y, x); + + // Check if the input associated to this label is seen by this pixel + auto it = pix.find(label); + if (it != pix.end()) { continue; } - PixelInfo & pix = graphCutInput(y, x); + // Look for another label in the neighboorhood which is seen by this pixel + bool modified = false; + for (int l = -1; l <= 1; l++) + { + int ny = y + l; + if (ny < 0 || ny >= labels.Height()) + { + continue; + } + + for (int c = -1; c <= 1; c++) + { + int nx = x + c; + if (nx < 0 || nx >= labels.Width()) + { + continue; + } + + IndexT otherLabel = labels(ny, nx); + if (otherLabel == label || otherLabel == UndefinedIndexT) + { + continue; + } + + // Check that this other label is seen by our pixel + const PixelInfo & pixOther = graphCutInput(ny, nx); + auto itOther = pixOther.find(otherLabel); + if (itOther == pixOther.end()) + { + continue; + } + + auto it = pix.find(otherLabel); + if (it != pix.end()) + { + labels(y, x) = otherLabel; + modified = true; + } + } + } - if (pix.size() == 0) + if (!modified) { - localLabels(y, x) = UndefinedIndexT; - continue; + ALICEVISION_LOG_ERROR("Invalid upscaling " << label << " " << x << " " << y); + return false; } + } + } - auto it = pix.find(label); - if (it == pix.end()) + return true; + } + + bool computeInputDistanceMap(image::Image & distanceMap, const image::Image & localLabels, IndexT inputId) + { + image::Image binarizedWorld(localLabels.Width(), localLabels.Height()); + + for(int y = 0; y < localLabels.Height(); y++) + { + for(int x = 0; x < localLabels.Width(); x++) + { + IndexT label = localLabels(y, x); + + if(label == inputId) { - localLabels(y, x) = pix.begin()->first; + binarizedWorld(y, x) = 1; + } + else + { + binarizedWorld(y, x) = 0; } } } + + image::Image seams(localLabels.Width(), localLabels.Height()); + if(!computeSeamsMap(seams, binarizedWorld)) + { + return false; + } + + if(!computeDistanceMap(distanceMap, seams)) + { + return false; + } + + return true; + } + + bool processInput(double & newCost, InputData & input, bool computeExpansion) + { + + //Get bounding box of input in panorama + //Dilate to have some pixels outside of the input + BoundingBox localBbox = input.rect.dilate(3); + localBbox.clampLeft(); + localBbox.clampTop(); + localBbox.clampBottom(_labels.getHeight() - 1); + + //Output must keep a margin also + BoundingBox outputBbox = input.rect; + outputBbox.left = input.rect.left - localBbox.left; + outputBbox.top = input.rect.top - localBbox.top; + + //Extract labels for the ROI + image::Image localLabels(localBbox.width, localBbox.height); + if (!loopyCachedImageExtract(localLabels, _labels, localBbox)) + { + return false; + } + + + //Build the input + image::Image graphCutInput(localBbox.width, localBbox.height, true); + if (!createInputOverlappingObservations(graphCutInput, localBbox)) + { + return false; + } + + // Fix upscaling induced bad labeling + if (!fixUpscaling(localLabels, graphCutInput)) + { + return false; + } + + // Backup update for upscaling + BoundingBox inputBb = localBbox; + inputBb.left = 0; + inputBb.top = 0; + if (!loopyCachedImageAssign(_labels, localLabels, localBbox, inputBb)) + { + return false; + } + + if (!computeExpansion) + { + return true; + } + + // Compute distance map to borders of the input seams + image::Image distanceMap(localLabels.Width(), localLabels.Height()); + if (!computeInputDistanceMap(distanceMap, localLabels, input.id)) + { + return false; + } + + //Remove pixels too far from seams + for (int i = 0; i < graphCutInput.Height(); i++) + { + for (int j = 0; j < graphCutInput.Width(); j++) + { + float d2 = float(distanceMap(i, j)); + float d = sqrt(d2); + + if (d2 > _maximal_distance_change + 1.0f) + { + graphCutInput(i, j).clear(); + } + } + } double oldCost = cost(localLabels, graphCutInput, input.id); if (!alphaExpansion(localLabels, distanceMap, graphCutInput, input.id)) @@ -473,15 +551,28 @@ class GraphcutSeams } else { - newCost = oldCost; + newCost = oldCost; } - return true; } - bool process() + bool process(bool computeAlphaExpansion) { + if (!computeAlphaExpansion) + { + for (auto & info : _inputs) + { + double cost; + if (!processInput(cost, info.second, false)) + { + return false; + } + } + + return true; + } + std::map costs; for (auto & info : _inputs) { @@ -498,7 +589,7 @@ class GraphcutSeams for (auto & info : _inputs) { double cost; - if (!processInput(cost, info.second)) + if (!processInput(cost, info.second, true)) { return false; } @@ -516,10 +607,6 @@ class GraphcutSeams } } - /*char filename[512]; - sprintf(filename, "/home/mmoc/test%d.exr", _outputWidth); - _labels.writeImage(filename);*/ - return true; } diff --git a/src/aliceVision/panorama/seams.cpp b/src/aliceVision/panorama/seams.cpp index d9e6b0414c..04a2e2e216 100644 --- a/src/aliceVision/panorama/seams.cpp +++ b/src/aliceVision/panorama/seams.cpp @@ -366,6 +366,11 @@ bool HierarchicalGraphcutSeams::setOriginalLabels(CachedImage& labels) return false; } + // + // Simply downscale the labels + // [a b e f ] + // [c d g h ] --> [a e] + for (int l = 1; l < _countLevels; l++) { @@ -521,7 +526,7 @@ bool HierarchicalGraphcutSeams::append(const aliceVision::image::Image Date: Wed, 18 Nov 2020 11:09:33 +0100 Subject: [PATCH 34/79] [panorama] Update graphcut maximal size --- src/aliceVision/panorama/seams.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/aliceVision/panorama/seams.cpp b/src/aliceVision/panorama/seams.cpp index 04a2e2e216..cacd595d1f 100644 --- a/src/aliceVision/panorama/seams.cpp +++ b/src/aliceVision/panorama/seams.cpp @@ -578,11 +578,11 @@ bool HierarchicalGraphcutSeams::process() } else { - _graphcuts[level].setMaximalDistance(100); + _graphcuts[level].setMaximalDistance(200); } bool computeAlphaExpansion = false; - if (w < 7500) + if (w < 5000) { computeAlphaExpansion = true; } From 30040ccd2098d6efcebb20d4f20c1ca4bfa9ea38 Mon Sep 17 00:00:00 2001 From: Fabien Date: Wed, 18 Nov 2020 16:45:06 +0100 Subject: [PATCH 35/79] [panorama] add a cap for pyramid --- src/aliceVision/panorama/laplacianCompositer.hpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/aliceVision/panorama/laplacianCompositer.hpp b/src/aliceVision/panorama/laplacianCompositer.hpp index 5d693c9159..fa198b1c70 100644 --- a/src/aliceVision/panorama/laplacianCompositer.hpp +++ b/src/aliceVision/panorama/laplacianCompositer.hpp @@ -10,10 +10,10 @@ namespace aliceVision class LaplacianCompositer : public Compositer { public: - LaplacianCompositer(image::TileCacheManager::shared_ptr & cacheManager, size_t outputWidth, size_t outputHeight, size_t bands) + LaplacianCompositer(image::TileCacheManager::shared_ptr & cacheManager, size_t outputWidth, size_t outputHeight) : Compositer(cacheManager, outputWidth, outputHeight) - , _pyramidPanorama(outputWidth, outputHeight, bands) - , _bands(bands) + , _pyramidPanorama(outputWidth, outputHeight, 1) + , _bands(1) { } @@ -67,7 +67,7 @@ class LaplacianCompositer : public Compositer //If the input scale is more important than previously processed, // The pyramid must be deepened accordingly - if(optimalLevelsCount > _bands) + if(optimalLevelsCount > _bands && _bands == 1) { _bands = optimalLevelsCount; if (!_pyramidPanorama.augment(_cacheManager, _bands)) From 0230ab00830e85ad7910bfdf1fade6c70446c49d Mon Sep 17 00:00:00 2001 From: Fabien Date: Wed, 18 Nov 2020 16:48:19 +0100 Subject: [PATCH 36/79] [panorama] update call to constructor --- src/software/pipeline/main_panoramaCompositing.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/software/pipeline/main_panoramaCompositing.cpp b/src/software/pipeline/main_panoramaCompositing.cpp index 7c0ebe48f4..b15ea5fea5 100644 --- a/src/software/pipeline/main_panoramaCompositing.cpp +++ b/src/software/pipeline/main_panoramaCompositing.cpp @@ -321,7 +321,7 @@ int aliceVision_main(int argc, char** argv) std::unique_ptr compositer; if (compositerType == "multiband") { - compositer = std::unique_ptr(new LaplacianCompositer(cacheManager, panoramaSize.first, panoramaSize.second, 1)); + compositer = std::unique_ptr(new LaplacianCompositer(cacheManager, panoramaSize.first, panoramaSize.second)); useSeams = true; useWeights = false; } From fb2a8c5d25880a7d3802513117d87d53ce1cea61 Mon Sep 17 00:00:00 2001 From: Fabien Date: Thu, 19 Nov 2020 08:30:16 +0100 Subject: [PATCH 37/79] [panorama] check with a new heuristic for graphcut --- src/aliceVision/panorama/graphcut.hpp | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/aliceVision/panorama/graphcut.hpp b/src/aliceVision/panorama/graphcut.hpp index 2f0e6ff7fc..f9bd917b1c 100644 --- a/src/aliceVision/panorama/graphcut.hpp +++ b/src/aliceVision/panorama/graphcut.hpp @@ -373,6 +373,7 @@ class GraphcutSeams // Look for another label in the neighboorhood which is seen by this pixel bool modified = false; + bool hadUndefined = false; for (int l = -1; l <= 1; l++) { int ny = y + l; @@ -390,11 +391,17 @@ class GraphcutSeams } IndexT otherLabel = labels(ny, nx); - if (otherLabel == label || otherLabel == UndefinedIndexT) + if (otherLabel == label) { continue; } + if (otherLabel == UndefinedIndexT) + { + hadUndefined = true; + continue; + } + // Check that this other label is seen by our pixel const PixelInfo & pixOther = graphCutInput(ny, nx); auto itOther = pixOther.find(otherLabel); @@ -414,8 +421,7 @@ class GraphcutSeams if (!modified) { - ALICEVISION_LOG_ERROR("Invalid upscaling " << label << " " << x << " " << y); - return false; + labels(y, x) = UndefinedIndexT; } } } From 211c1e8f9fec705169b5f9852f4e7dbc2795fc17 Mon Sep 17 00:00:00 2001 From: Fabien Date: Fri, 20 Nov 2020 12:29:32 +0100 Subject: [PATCH 38/79] [panorama] change some constants --- src/aliceVision/panorama/compositer.hpp | 2 +- .../pipeline/main_panoramaCompositing.cpp | 18 ++---------------- 2 files changed, 3 insertions(+), 17 deletions(-) diff --git a/src/aliceVision/panorama/compositer.hpp b/src/aliceVision/panorama/compositer.hpp index fa1a730983..a017c34d0e 100644 --- a/src/aliceVision/panorama/compositer.hpp +++ b/src/aliceVision/panorama/compositer.hpp @@ -119,7 +119,7 @@ class Compositer virtual size_t getOptimalScale(int width, int height) const { - return 1; + return 0; } virtual int getBorderSize() const diff --git a/src/software/pipeline/main_panoramaCompositing.cpp b/src/software/pipeline/main_panoramaCompositing.cpp index b15ea5fea5..4f6bcd5a02 100644 --- a/src/software/pipeline/main_panoramaCompositing.cpp +++ b/src/software/pipeline/main_panoramaCompositing.cpp @@ -125,7 +125,7 @@ bool computeGCLabels(CachedImage & labels, image::TileCacheManager::shar { ALICEVISION_LOG_INFO("Estimating smart seams for panorama"); - int pyramidSize = std::max(0, smallestViewScale - 1); + int pyramidSize = 1 + std::max(0, smallestViewScale - 1); ALICEVISION_LOG_INFO("Graphcut pyramid size is " << pyramidSize); HierarchicalGraphcutSeams seams(cacheManager, panoramaSize.first, panoramaSize.second, pyramidSize); @@ -307,7 +307,7 @@ int aliceVision_main(int argc, char** argv) } // Configure the cache manager memory - cacheManager->setInCoreMaxObjectCount(1000); + cacheManager->setInCoreMaxObjectCount(1000 * 100); if (overlayType == "borders" || overlayType == "all") { @@ -392,20 +392,6 @@ int aliceVision_main(int argc, char** argv) ALICEVISION_LOG_INFO(viewOrderedByScale.size() << " views to process"); - /*BoundingBoxPanoramaMap map(panoramaSize.first, panoramaSize.second); - if (!buildMap(map, viewOrderedByScale, warpingFolder, compositer)) - { - ALICEVISION_LOG_ERROR("Error computing map"); - return EXIT_FAILURE; - } - - std::list bestInitials; - if (!map.getBestInitial(bestInitials)) - { - ALICEVISION_LOG_ERROR("Error computing best elements"); - return EXIT_SUCCESS; - }*/ - CachedImage labels; if (useSeams) { From 93cd1a675c0f8eb3b23fe47059e4c848116ef87e Mon Sep 17 00:00:00 2001 From: Fabien Castan Date: Tue, 8 Dec 2020 11:59:03 +0100 Subject: [PATCH 39/79] [system] new availableRam information --- src/aliceVision/system/MemoryInfo.cpp | 45 ++++++++++++++++++++++----- src/aliceVision/system/MemoryInfo.hpp | 13 ++++---- 2 files changed, 45 insertions(+), 13 deletions(-) diff --git a/src/aliceVision/system/MemoryInfo.cpp b/src/aliceVision/system/MemoryInfo.cpp index fe02b7b352..452f53a506 100644 --- a/src/aliceVision/system/MemoryInfo.cpp +++ b/src/aliceVision/system/MemoryInfo.cpp @@ -15,6 +15,8 @@ #include #elif defined(__LINUX__) #include +#include +#include #elif defined(__APPLE__) #include #include @@ -31,6 +33,28 @@ namespace aliceVision { namespace system { +#if defined(__LINUX__) +unsigned long linuxGetAvailableRam() +{ + std::string token; + std::ifstream file("/proc/meminfo"); + while(file >> token) { + if(token == "MemAvailable:") { + unsigned long mem; + if(file >> mem) { + // read in kB and convert to bytes + return mem * 1024; + } else { + return 0; + } + } + // ignore rest of the line + file.ignore(std::numeric_limits::max(), '\n'); + } + return 0; // nothing found +} +#endif + MemoryInfo getMemoryInfo() { MemoryInfo infos; @@ -41,7 +65,7 @@ MemoryInfo getMemoryInfo() // memory.dwMemoryLoad; infos.totalRam = memory.dwTotalPhys; - infos.freeRam = memory.dwAvailPhys; + infos.availableRam = infos.freeRam = memory.dwAvailPhys; // memory.dwTotalPageFile; // memory.dwAvailPageFile; infos.totalSwap = memory.dwTotalVirtual; @@ -52,6 +76,11 @@ MemoryInfo getMemoryInfo() infos.totalRam = sys_info.totalram * sys_info.mem_unit; infos.freeRam = sys_info.freeram * sys_info.mem_unit; + + infos.availableRam = linuxGetAvailableRam(); + if(infos.availableRam == 0) + infos.availableRam = infos.freeRam; + // infos.sharedRam = sys_info.sharedram * sys_info.mem_unit; // infos.bufferRam = sys_info.bufferram * sys_info.mem_unit; infos.totalSwap = sys_info.totalswap * sys_info.mem_unit; @@ -89,11 +118,12 @@ MemoryInfo getMemoryInfo() // (int64_t)page_size; // infos.freeRam = infos.totalRam - used; infos.freeRam = (int64_t)vm_stat.free_count * (int64_t)page_size; + infos.availableRam = infos.freeRam; } #else // TODO: could be done on FreeBSD too // see https://github.com/xbmc/xbmc/blob/master/xbmc/linux/XMemUtils.cpp - infos.totalRam = infos.freeRam = infos.totalSwap = infos.freeSwap = std::numeric_limits::max(); + infos.totalRam = infos.freeRam = infos.availableRam = infos.totalSwap = infos.freeSwap = std::numeric_limits::max(); #endif return infos; @@ -101,12 +131,13 @@ MemoryInfo getMemoryInfo() std::ostream& operator<<(std::ostream& os, const MemoryInfo& infos) { - const float convertionGb = std::pow(2,30); + const double convertionGb = std::pow(2,30); os << std::setw(5) - << "\t- Total RAM: " << (infos.totalRam / convertionGb) << " GB" << std::endl - << "\t- Free RAM: " << (infos.freeRam / convertionGb) << " GB" << std::endl - << "\t- Total swap: " << (infos.totalSwap / convertionGb) << " GB" << std::endl - << "\t- Free swap: " << (infos.freeSwap / convertionGb) << " GB" << std::endl; + << "\t- Total RAM: " << (infos.totalRam / convertionGb) << " GB" << std::endl + << "\t- Free RAM: " << (infos.freeRam / convertionGb) << " GB" << std::endl + << "\t- Available RAM: " << (infos.availableRam / convertionGb) << " GB" << std::endl + << "\t- Total swap: " << (infos.totalSwap / convertionGb) << " GB" << std::endl + << "\t- Free swap: " << (infos.freeSwap / convertionGb) << " GB" << std::endl; return os; } diff --git a/src/aliceVision/system/MemoryInfo.hpp b/src/aliceVision/system/MemoryInfo.hpp index c88041d7d2..4ecc2aeabf 100644 --- a/src/aliceVision/system/MemoryInfo.hpp +++ b/src/aliceVision/system/MemoryInfo.hpp @@ -14,12 +14,13 @@ namespace system { struct MemoryInfo { - std::size_t totalRam; - std::size_t freeRam; - // std::size_t sharedRam; - // std::size_t bufferRam; - std::size_t totalSwap; - std::size_t freeSwap; + std::size_t totalRam{0}; + std::size_t freeRam{0}; + std::size_t availableRam{0}; + // std::size_t sharedRam{0}; + // std::size_t bufferRam{0}; + std::size_t totalSwap{0}; + std::size_t freeSwap{0}; }; MemoryInfo getMemoryInfo(); From 15efca8f9ee0a7c9e12dc1c86f21053025d666ba Mon Sep 17 00:00:00 2001 From: Fabien Castan Date: Tue, 8 Dec 2020 11:59:49 +0100 Subject: [PATCH 40/79] [software] new utility hardwareResources --- src/software/utils/CMakeLists.txt | 11 ++ src/software/utils/main_hardwareResources.cpp | 102 ++++++++++++++++++ 2 files changed, 113 insertions(+) create mode 100644 src/software/utils/main_hardwareResources.cpp diff --git a/src/software/utils/CMakeLists.txt b/src/software/utils/CMakeLists.txt index 828b18a8ac..9ac628e7d8 100644 --- a/src/software/utils/CMakeLists.txt +++ b/src/software/utils/CMakeLists.txt @@ -4,6 +4,17 @@ # Software PROPERTY FOLDER is 'Software/Utils' set(FOLDER_SOFTWARE_UTILS "Software/Utils") + +if(ALICEVISION_HAVE_CUDA) +alicevision_add_software(aliceVision_utils_hardwareResources + SOURCE main_hardwareResources.cpp + FOLDER ${FOLDER_SOFTWARE_UTILS} + LINKS aliceVision_system + aliceVision_gpu + Boost::program_options +) +endif() + if(ALICEVISION_BUILD_SFM) # Uncertainty if(ALICEVISION_HAVE_UNCERTAINTYTE) diff --git a/src/software/utils/main_hardwareResources.cpp b/src/software/utils/main_hardwareResources.cpp new file mode 100644 index 0000000000..6e995658b8 --- /dev/null +++ b/src/software/utils/main_hardwareResources.cpp @@ -0,0 +1,102 @@ +// This file is part of the AliceVision project. +// Copyright (c) 2020 AliceVision contributors. +// This Source Code Form is subject to the terms of the Mozilla Public License, +// v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at https://mozilla.org/MPL/2.0/. + +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include + + +// These constants define the current software version. +// They must be updated when the command line is changed. +#define ALICEVISION_SOFTWARE_VERSION_MAJOR 1 +#define ALICEVISION_SOFTWARE_VERSION_MINOR 0 + +using namespace aliceVision; + +namespace po = boost::program_options; + + +int aliceVision_main(int argc, char **argv) +{ + // command-line parameters + std::string verboseLevel = system::EVerboseLevel_enumToString(system::Logger::getDefaultVerboseLevel()); + + po::options_description allParams("AliceVision hardwareResources"); + + po::options_description logParams("Log parameters"); + logParams.add_options() + ("verboseLevel,v", po::value(&verboseLevel)->default_value(verboseLevel), + "verbosity level (fatal, error, warning, info, debug, trace)."); + + allParams.add(logParams); + + po::variables_map vm; + try + { + po::store(po::parse_command_line(argc, argv, allParams), vm); + + if(vm.count("help")) + { + ALICEVISION_COUT(allParams); + return EXIT_SUCCESS; + } + po::notify(vm); + } + catch(boost::program_options::required_option& e) + { + ALICEVISION_CERR("ERROR: " << e.what()); + ALICEVISION_COUT("Usage:\n\n" << allParams); + return EXIT_FAILURE; + } + catch(boost::program_options::error& e) + { + ALICEVISION_CERR("ERROR: " << e.what()); + ALICEVISION_COUT("Usage:\n\n" << allParams); + return EXIT_FAILURE; + } + + ALICEVISION_COUT("Program called with the following parameters:"); + ALICEVISION_COUT(vm); + + // set verbose level + system::Logger::get()->setLogLevel(verboseLevel); + + // print GPU Information + ALICEVISION_LOG_INFO(gpu::gpuInformationCUDA()); + + system::MemoryInfo memoryInformation = system::getMemoryInfo(); + + ALICEVISION_LOG_INFO("Memory information: " << std::endl << memoryInformation); + + if(memoryInformation.availableRam == 0) + { + ALICEVISION_LOG_WARNING("Cannot find available system memory, this can be due to OS limitation.\n" + "Use only one thread for CPU feature extraction."); + } + else + { + const double oneGB = 1024.0 * 1024.0 * 1024.0; + if(memoryInformation.availableRam < 0.5 * memoryInformation.totalRam) + { + ALICEVISION_LOG_WARNING("More than half of the RAM is used by other applications. It would be more efficient to close them."); + ALICEVISION_LOG_WARNING(" => " + << std::size_t(std::round(double(memoryInformation.totalRam - memoryInformation.availableRam) / oneGB)) + << " GB are used by other applications for a total RAM capacity of " + << std::size_t(std::round(double(memoryInformation.totalRam) / oneGB)) << " GB."); + } + } + + return EXIT_SUCCESS; +} From 5d076344cded6ee1d4bd75436d7cf1ab0476f8e4 Mon Sep 17 00:00:00 2001 From: Fabien Castan Date: Tue, 8 Dec 2020 12:02:56 +0100 Subject: [PATCH 41/79] [software] PanoramaCompositing: set max memory available to cache - cache: add trace log for read/write to cache files - PanoramaCompositing: create cache folder if it does not already exist --- src/aliceVision/image/cache.cpp | 67 ++++++++++++------- src/aliceVision/image/cache.hpp | 12 +++- .../pipeline/main_panoramaCompositing.cpp | 29 +++++--- 3 files changed, 72 insertions(+), 36 deletions(-) diff --git a/src/aliceVision/image/cache.cpp b/src/aliceVision/image/cache.cpp index 87d66b2cb4..8ed7689d54 100644 --- a/src/aliceVision/image/cache.cpp +++ b/src/aliceVision/image/cache.cpp @@ -1,6 +1,10 @@ #include "cache.hpp" + +#include + #include + namespace aliceVision { namespace image @@ -30,6 +34,11 @@ void CacheManager::wipe() { _nextObjectId = 0; } + +void CacheManager::setMaxMemory(size_t maxMemorySize) { + _incoreBlockUsageMax = maxMemorySize / _blockSize; +} + void CacheManager::setInCoreMaxObjectCount(size_t max) { _incoreBlockUsageMax = max; } @@ -50,16 +59,25 @@ std::string CacheManager::getPathForIndex(size_t indexId) { void CacheManager::deleteIndexFiles() { - /* Remove all cache files */ - for (std::pair & p : _indexPaths) { + std::size_t cacheSize = 0; + for (std::pair & p : _indexPaths) + { + const std::size_t s = boost::filesystem::file_size(p.second); + ALICEVISION_LOG_TRACE("CacheManager::deleteIndexFiles: '" << p.second << "': " << s / (1024*1024) << "MB."); + cacheSize += s; + } + ALICEVISION_LOG_DEBUG("CacheManager::deleteIndexFiles: cache size is " << cacheSize / (1024*1024) << "MB."); + // Remove all cache files + for (std::pair & p : _indexPaths) + { boost::filesystem::path path(p.second); boost::filesystem::remove(path); } - /* Remove list of cache files */ + // Remove list of cache files _indexPaths.clear(); -} +} bool CacheManager::prepareBlockGroup(size_t startBlockId, size_t blocksCount) { @@ -82,7 +100,9 @@ bool CacheManager::prepareBlockGroup(size_t startBlockId, size_t blocksCount) { if (!file_index.is_open()) { return false; } - + + ALICEVISION_LOG_TRACE("CacheManager::prepareBlockGroup: " << blocksCount * _blockSize << " bytes to '" << path << "'."); + /*write a dummy byte at the end of the tile to "book" this place on disk*/ file_index.seekp(position_in_index + len - 1, file_index.beg); if (!file_index) { @@ -98,12 +118,12 @@ bool CacheManager::prepareBlockGroup(size_t startBlockId, size_t blocksCount) { std::unique_ptr CacheManager::load(size_t startBlockId, size_t blockCount) { - size_t indexId = startBlockId / _blockCountPerIndex; - size_t blockIdInIndex = startBlockId % _blockCountPerIndex; - size_t positionInIndex = blockIdInIndex * _blockSize; - size_t groupLength = _blockSize * blockCount; + const size_t indexId = startBlockId / _blockCountPerIndex; + const size_t blockIdInIndex = startBlockId % _blockCountPerIndex; + const size_t positionInIndex = blockIdInIndex * _blockSize; + const size_t groupLength = _blockSize * blockCount; - std::string path = getPathForIndex(indexId); + const std::string path = getPathForIndex(indexId); std::ifstream file_index(path, std::ios::binary); if (!file_index.is_open()) { @@ -115,6 +135,8 @@ std::unique_ptr CacheManager::load(size_t startBlockId, size_t bl return std::unique_ptr(); } + ALICEVISION_LOG_TRACE("CacheManager::load: read " << groupLength << " bytes from '" << path << "' at position " << positionInIndex << "."); + std::unique_ptr data(new unsigned char[groupLength]); file_index.read(reinterpret_cast(data.get()), groupLength); if (!file_index) { @@ -126,29 +148,30 @@ std::unique_ptr CacheManager::load(size_t startBlockId, size_t bl bool CacheManager::save(std::unique_ptr && data, size_t startBlockId, size_t blockCount) { - size_t indexId = startBlockId / _blockCountPerIndex; - size_t blockIdInIndex = startBlockId % _blockCountPerIndex; - size_t positionInIndex = blockIdInIndex * _blockSize; - size_t groupLength = _blockSize * blockCount; + const size_t indexId = startBlockId / _blockCountPerIndex; + const size_t blockIdInIndex = startBlockId % _blockCountPerIndex; + const size_t positionInIndex = blockIdInIndex * _blockSize; + const size_t groupLength = _blockSize * blockCount; + + const std::string path = getPathForIndex(indexId); - std::string path = getPathForIndex(indexId); - std::ofstream file_index(path, std::ios::binary | std::ios::out | std::ios::in); if (!file_index.is_open()) { return false; - } + } file_index.seekp(positionInIndex, std::ios::beg); if (file_index.fail()) { return false; } - + const unsigned char * bytesToWrite = data.get(); if (bytesToWrite == nullptr) { return false; } else { - /*Write data*/ + // Write data + ALICEVISION_LOG_TRACE("CacheManager::save: write " << groupLength << " bytes to '" << path << "' at position " << positionInIndex << "."); file_index.write(reinterpret_cast(bytesToWrite), groupLength); } @@ -203,7 +226,7 @@ bool CacheManager::acquireObject(std::unique_ptr & data, size_t o item.objectId = objectId; item.objectSize = memitem.countBlock; - /* Check mur */ + /* Check mru */ std::pair p = _mru.push_front(item); if (p.second) { @@ -243,8 +266,6 @@ bool CacheManager::acquireObject(std::unique_ptr & data, size_t o onRemovedFromMRU(item.objectId); } - - return true; } @@ -429,4 +450,4 @@ void TileCacheManager::onRemovedFromMRU(size_t objectId) { } } -} \ No newline at end of file +} diff --git a/src/aliceVision/image/cache.hpp b/src/aliceVision/image/cache.hpp index c53cd1239f..a84b0b28a1 100644 --- a/src/aliceVision/image/cache.hpp +++ b/src/aliceVision/image/cache.hpp @@ -81,7 +81,7 @@ class CachedTile { /* Tells the system that we need the data for this tile. - This means that is the data is out of core, we want it back + This means that the data is out of core, we want it back. @return false if the process failed to grab data. */ bool acquire(); @@ -186,12 +186,18 @@ class CacheManager { CacheManager(const std::string & pathStorage, size_t blockSize, size_t maxBlocksPerIndex); virtual ~CacheManager(); + /** + * Set the maximal memory size + * @param max the maximal memory size + */ + void setMaxMemory(size_t maxMemorySize); + /** * Set the maximal number number of items simultaneously in core * @param max the maximal number of items */ void setInCoreMaxObjectCount(size_t max); - + /** * Create a new object of size block count * @param objectId the created object index @@ -330,4 +336,4 @@ class TileCacheManager : public CacheManager, public std::enable_shared_from_thi }; } -} \ No newline at end of file +} diff --git a/src/software/pipeline/main_panoramaCompositing.cpp b/src/software/pipeline/main_panoramaCompositing.cpp index 4f6bcd5a02..0c94a2e1d2 100644 --- a/src/software/pipeline/main_panoramaCompositing.cpp +++ b/src/software/pipeline/main_panoramaCompositing.cpp @@ -4,15 +4,23 @@ // v. 2.0. If a copy of the MPL was not distributed with this file, // You can obtain one at https://mozilla.org/MPL/2.0/. +#include +#include +#include +#include +#include +#include + // Input and geometry #include #include -// Image stuff +// Image #include #include -// Logging stuff +// System +#include #include // Reading command line options @@ -27,13 +35,6 @@ #include #include -#include -#include -#include -#include -#include -#include - // These constants define the current software version. // They must be updated when the command line is changed. #define ALICEVISION_SOFTWARE_VERSION_MAJOR 1 @@ -298,6 +299,11 @@ int aliceVision_main(int argc, char** argv) ALICEVISION_LOG_INFO("Output panorama size set to " << panoramaSize.first << "x" << panoramaSize.second); } + if(!temporaryCachePath.empty() && !fs::exists(temporaryCachePath)) + { + fs::create_directory(temporaryCachePath); + } + // Create a cache manager image::TileCacheManager::shared_ptr cacheManager = image::TileCacheManager::create(temporaryCachePath, 256, 256, 65536); if(!cacheManager) @@ -307,7 +313,10 @@ int aliceVision_main(int argc, char** argv) } // Configure the cache manager memory - cacheManager->setInCoreMaxObjectCount(1000 * 100); + system::MemoryInfo memInfo = system::getMemoryInfo(); + const double convertionGb = std::pow(2,30); + ALICEVISION_LOG_INFO("Available RAM is " << std::setw(5) << memInfo.availableRam / convertionGb << "GB (" << memInfo.availableRam << " octets)."); + cacheManager->setMaxMemory(memInfo.freeRam); if (overlayType == "borders" || overlayType == "all") { From 7952e169dd5cf8774e8c22f48041b144bfd15570 Mon Sep 17 00:00:00 2001 From: Fabien Castan Date: Tue, 8 Dec 2020 12:03:44 +0100 Subject: [PATCH 42/79] [software] texturing: use availableRam (instead of freeRam) --- src/aliceVision/mesh/Texturing.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/aliceVision/mesh/Texturing.cpp b/src/aliceVision/mesh/Texturing.cpp index cca7c892de..1563f51fbc 100644 --- a/src/aliceVision/mesh/Texturing.cpp +++ b/src/aliceVision/mesh/Texturing.cpp @@ -269,8 +269,8 @@ void Texturing::generateTextures(const mvsUtils::MultiViewParams& mp, const std::size_t atlasContribMemSize = texParams.textureSide * texParams.textureSide * (sizeof(Color)+sizeof(float)) / std::pow(2,20); //MB const std::size_t atlasPyramidMaxMemSize = texParams.nbBand * atlasContribMemSize; - const int freeRam = int(memInfo.freeRam / std::pow(2,20)); - const int availableMem = freeRam - 2 * imageMaxMemSize - imagePyramidMaxMemSize; // keep some memory for the 2 input images in cache and one laplacian pyramid + const int availableRam = int(memInfo.availableRam / std::pow(2,20)); + const int availableMem = availableRam - 2 * imageMaxMemSize - imagePyramidMaxMemSize; // keep some memory for the 2 input images in cache and one laplacian pyramid const int nbAtlas = _atlases.size(); int nbAtlasMax = std::floor(availableMem / atlasPyramidMaxMemSize); //maximum number of textures laplacian pyramid in RAM @@ -279,8 +279,8 @@ void Texturing::generateTextures(const mvsUtils::MultiViewParams& mp, nbAtlasMax -= 1; nbAtlasMax = std::max(1, nbAtlasMax); //if not enough memory, do it one by one - ALICEVISION_LOG_INFO("Total amount of free RAM : " << freeRam << " MB."); - ALICEVISION_LOG_INFO("Total amount of memory available : " << availableMem << " MB."); + ALICEVISION_LOG_INFO("Total amount of available RAM : " << availableRam << " MB."); + ALICEVISION_LOG_INFO("Total amount of memory remaining for the computation : " << availableMem << " MB."); ALICEVISION_LOG_INFO("Total amount of an image in memory : " << imageMaxMemSize << " MB."); ALICEVISION_LOG_INFO("Total amount of an atlas pyramid in memory: " << atlasPyramidMaxMemSize << " MB."); ALICEVISION_LOG_INFO("Processing " << nbAtlas << " atlases by chunks of " << nbAtlasMax); From 960aca1fe59f9cb3633bc777d29abde457d8f375 Mon Sep 17 00:00:00 2001 From: Fabien Castan Date: Tue, 8 Dec 2020 12:04:12 +0100 Subject: [PATCH 43/79] [software] featureExtraction: use availableRam (instead of freeRam) --- src/software/pipeline/main_featureExtraction.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/software/pipeline/main_featureExtraction.cpp b/src/software/pipeline/main_featureExtraction.cpp index 7aa111d6c6..c514ddb72f 100644 --- a/src/software/pipeline/main_featureExtraction.cpp +++ b/src/software/pipeline/main_featureExtraction.cpp @@ -174,34 +174,34 @@ class FeatureExtractor // How many buffers can fit in 90% of the available RAM? // This is used to estimate how many jobs can be computed in parallel without SWAP. - const std::size_t memoryImageCapacity = std::size_t((0.9 * memoryInformation.freeRam) / jobMaxMemoryConsuption); + const std::size_t memoryImageCapacity = std::size_t((0.9 * memoryInformation.availableRam) / jobMaxMemoryConsuption); std::size_t nbThreads = std::max(std::size_t(1), memoryImageCapacity); ALICEVISION_LOG_INFO("Max number of threads regarding memory usage: " << nbThreads); const double oneGB = 1024.0 * 1024.0 * 1024.0; - if(jobMaxMemoryConsuption > memoryInformation.freeRam) + if(jobMaxMemoryConsuption > memoryInformation.availableRam) { ALICEVISION_LOG_WARNING("The amount of RAM available is critical to extract features."); if(jobMaxMemoryConsuption <= memoryInformation.totalRam) { ALICEVISION_LOG_WARNING("But the total amount of RAM is enough to extract features, so you should close other running applications."); - ALICEVISION_LOG_WARNING(" => " << std::size_t(std::round((double(memoryInformation.totalRam - memoryInformation.freeRam) / oneGB))) + ALICEVISION_LOG_WARNING(" => " << std::size_t(std::round((double(memoryInformation.totalRam - memoryInformation.availableRam) / oneGB))) << " GB are used by other applications for a total RAM capacity of " << std::size_t(std::round(double(memoryInformation.totalRam) / oneGB)) << " GB."); } } else { - if(memoryInformation.freeRam < 0.5 * memoryInformation.totalRam) + if(memoryInformation.availableRam < 0.5 * memoryInformation.totalRam) { ALICEVISION_LOG_WARNING("More than half of the RAM is used by other applications. It would be more efficient to close them."); ALICEVISION_LOG_WARNING(" => " - << std::size_t(std::round(double(memoryInformation.totalRam - memoryInformation.freeRam) / oneGB)) + << std::size_t(std::round(double(memoryInformation.totalRam - memoryInformation.availableRam) / oneGB)) << " GB are used by other applications for a total RAM capacity of " << std::size_t(std::round(double(memoryInformation.totalRam) / oneGB)) << " GB."); } } - if(memoryInformation.freeRam == 0) + if(memoryInformation.availableRam == 0) { ALICEVISION_LOG_WARNING("Cannot find available system memory, this can be due to OS limitation.\n" "Use only one thread for CPU feature extraction."); From e61de70b69eb6d24beba764353a3e63aa141f114 Mon Sep 17 00:00:00 2001 From: Fabien Servant Date: Fri, 11 Dec 2020 08:53:58 +0100 Subject: [PATCH 44/79] remove some useless computations and memory access --- src/aliceVision/panorama/graphcut.hpp | 118 +++++++++++++++++--------- 1 file changed, 77 insertions(+), 41 deletions(-) diff --git a/src/aliceVision/panorama/graphcut.hpp b/src/aliceVision/panorama/graphcut.hpp index f9bd917b1c..685e6023ad 100644 --- a/src/aliceVision/panorama/graphcut.hpp +++ b/src/aliceVision/panorama/graphcut.hpp @@ -132,6 +132,7 @@ class MaxFlow_AdjList const NodeType _T; //< fullness }; + bool computeSeamsMap(image::Image& seams, const image::Image& labels); class GraphcutSeams @@ -145,7 +146,8 @@ class GraphcutSeams CachedImage mask; }; - using PixelInfo = std::unordered_map; + using IndexedColor = std::pair; + using PixelInfo = std::vector; public: @@ -157,6 +159,32 @@ class GraphcutSeams virtual ~GraphcutSeams() = default; + std::pair findIndex(const PixelInfo & pix, IndexT id) + { + for (int i = 0; i < pix.size(); i++) + { + if (pix[i].first == id) + { + return pix[i]; + } + } + + return std::make_pair(UndefinedIndexT, image::RGBfColor(0.0f)); + } + + bool existIndex(const PixelInfo & pix, IndexT id) + { + for (int i = 0; i < pix.size(); i++) + { + if (pix[i].first == id) + { + return true; + } + } + + return false; + } + bool setOriginalLabels(CachedImage & existing_labels) { if (!_labels.deepCopy(existing_labels)) @@ -211,7 +239,7 @@ class GraphcutSeams _maximal_distance_change = dist; } - bool createInputOverlappingObservations(image::Image & graphCutInput, const BoundingBox & interestBbox) + bool createInputOverlappingObservations(image::Image & graphCutInput, const BoundingBox & interestBbox, bool loadColors) { for (auto & otherInput : _inputs) { @@ -235,9 +263,12 @@ class GraphcutSeams } image::Image otherColor(otherInputBbox.width, otherInputBbox.height); - if (!otherInput.second.color.extract(otherColor, otherInputBbox, otherInputBbox)) + if (loadColors) { - return false; + if (!otherInput.second.color.extract(otherColor, otherInputBbox, otherInputBbox)) + { + return false; + } } image::Image otherMask(otherInputBbox.width, otherInputBbox.height); @@ -273,7 +304,7 @@ class GraphcutSeams } PixelInfo & pix = graphCutInput(y_current, x_current); - pix[otherInput.first] = otherColor(y_other, x_other); + pix.push_back(std::make_pair(otherInput.first, otherColor(y_other, x_other))); } } } @@ -305,7 +336,7 @@ class GraphcutSeams } PixelInfo & pix = graphCutInput(y_current, x_current); - pix[otherInput.first] = otherColor(y_other, x_other); + pix.push_back(std::make_pair(otherInput.first, otherColor(y_other, x_other))); } } } @@ -337,7 +368,7 @@ class GraphcutSeams } PixelInfo & pix = graphCutInput(y_current, x_current); - pix[otherInput.first] = otherColor(y_other, x_other); + pix.push_back(std::make_pair(otherInput.first, otherColor(y_other, x_other))); } } } @@ -365,8 +396,8 @@ class GraphcutSeams const PixelInfo & pix = graphCutInput(y, x); // Check if the input associated to this label is seen by this pixel - auto it = pix.find(label); - if (it != pix.end()) + auto it = existIndex(pix, label); + if (it) { continue; } @@ -404,14 +435,14 @@ class GraphcutSeams // Check that this other label is seen by our pixel const PixelInfo & pixOther = graphCutInput(ny, nx); - auto itOther = pixOther.find(otherLabel); - if (itOther == pixOther.end()) + auto itOther = existIndex(pixOther, otherLabel); + if (!itOther) { continue; } - auto it = pix.find(otherLabel); - if (it != pix.end()) + auto it = existIndex(pix, otherLabel); + if (it) { labels(y, x) = otherLabel; modified = true; @@ -488,12 +519,14 @@ class GraphcutSeams //Build the input + ALICEVISION_LOG_INFO("build input"); image::Image graphCutInput(localBbox.width, localBbox.height, true); - if (!createInputOverlappingObservations(graphCutInput, localBbox)) + if (!createInputOverlappingObservations(graphCutInput, localBbox, computeExpansion)) { return false; } + ALICEVISION_LOG_INFO("fix input"); // Fix upscaling induced bad labeling if (!fixUpscaling(localLabels, graphCutInput)) { @@ -501,6 +534,7 @@ class GraphcutSeams } // Backup update for upscaling + ALICEVISION_LOG_INFO("save input"); BoundingBox inputBb = localBbox; inputBb.left = 0; inputBb.top = 0; @@ -570,6 +604,8 @@ class GraphcutSeams for (auto & info : _inputs) { double cost; + ALICEVISION_LOG_INFO("process input"); + if (!processInput(cost, info.second, false)) { return false; @@ -654,53 +690,53 @@ class GraphcutSeams bool hasCLY = false; - auto it = input(y, x).find(label); - if (it != input(y, x).end()) + auto it = findIndex(input(y, x), label); + if (it.first != UndefinedIndexT) { hasCLC = true; - CColorLC = it->second; + CColorLC = it.second; } - it = input(y, x).find(labelx); - if (it != input(y, x).end()) + it = findIndex(input(y, x), labelx); + if (it.first != UndefinedIndexT) { hasCLX = true; - CColorLX = it->second; + CColorLX = it.second; } - it = input(y, x).find(labely); - if (it != input(y, x).end()) + it = findIndex(input(y, x), labely); + if (it.first != UndefinedIndexT) { hasCLY = true; - CColorLY = it->second; + CColorLY = it.second; } - it = input(y, xp).find(label); - if (it != input(y, xp).end()) + it = findIndex(input(y, xp), label); + if (it.first != UndefinedIndexT) { hasXLC = true; - XColorLC = it->second; + XColorLC = it.second; } - it = input(y, xp).find(labelx); - if (it != input(y, xp).end()) + it = findIndex(input(y, xp), labelx); + if (it.first != UndefinedIndexT) { hasXLX = true; - XColorLX = it->second; + XColorLX = it.second; } - it = input(yp, x).find(label); - if (it != input(yp, x).end()) + it = findIndex(input(yp, x), label); + if (it.first != UndefinedIndexT) { hasYLC = true; - YColorLC = it->second; + YColorLC = it.second; } - it = input(yp, x).find(labely); - if (it != input(yp, x).end()) + it = findIndex(input(yp, x), labely); + if (it.first != UndefinedIndexT) { hasYLY = true; - YColorLY = it->second; + YColorLY = it.second; } if(!hasCLC || !hasXLX || !hasYLY) @@ -764,19 +800,19 @@ class GraphcutSeams image::RGBfColor currentColor; image::RGBfColor otherColor; - auto it = input(y, x).find(currentLabel); - if (it != input(y, x).end()) + auto it = findIndex(input(y, x), currentLabel); + if (it.first != UndefinedIndexT) { - currentColor = it->second; + currentColor = it.second; mask(y, x) = 1; } if (label != currentLabel) { - it = input(y, x).find(label); - if (it != input(y, x).end()) + auto it = findIndex(input(y, x), label); + if (it.first != UndefinedIndexT) { - otherColor = it->second; + otherColor = it.second; if (dist > _maximal_distance_change) { From 2dfce7cb1a74026ffcf63250137f1b87ede13b5d Mon Sep 17 00:00:00 2001 From: Fabien Date: Mon, 14 Dec 2020 14:08:15 +0100 Subject: [PATCH 45/79] [panorama] try new method for large panoramas --- src/aliceVision/panorama/CMakeLists.txt | 4 +- src/aliceVision/panorama/alphaCompositer.hpp | 58 +-- src/aliceVision/panorama/boundingBox.hpp | 12 +- .../panorama/boundingBoxPanoramaMap.cpp | 167 ------- .../panorama/boundingBoxPanoramaMap.hpp | 37 -- src/aliceVision/panorama/compositer.hpp | 103 ++--- src/aliceVision/panorama/graphcut.hpp | 3 - src/aliceVision/panorama/imageOps.cpp | 19 +- src/aliceVision/panorama/imageOps.hpp | 14 +- .../panorama/laplacianCompositer.hpp | 39 +- src/aliceVision/panorama/laplacianPyramid.cpp | 362 ++++----------- src/aliceVision/panorama/laplacianPyramid.hpp | 22 +- src/aliceVision/panorama/panoramaMap.cpp | 112 +++++ src/aliceVision/panorama/panoramaMap.hpp | 63 +++ src/aliceVision/panorama/remapBbox.cpp | 2 - src/aliceVision/panorama/seams.cpp | 117 ++--- src/aliceVision/panorama/seams.hpp | 16 +- src/software/pipeline/CMakeLists.txt | 10 + .../pipeline/main_panoramaCompositing.cpp | 436 +++++------------- src/software/pipeline/main_panoramaSeams.cpp | 356 ++++++++++++++ 20 files changed, 885 insertions(+), 1067 deletions(-) delete mode 100644 src/aliceVision/panorama/boundingBoxPanoramaMap.cpp delete mode 100644 src/aliceVision/panorama/boundingBoxPanoramaMap.hpp create mode 100644 src/aliceVision/panorama/panoramaMap.cpp create mode 100644 src/aliceVision/panorama/panoramaMap.hpp create mode 100644 src/software/pipeline/main_panoramaSeams.cpp diff --git a/src/aliceVision/panorama/CMakeLists.txt b/src/aliceVision/panorama/CMakeLists.txt index aa7d2fa4bf..2ee176a39b 100644 --- a/src/aliceVision/panorama/CMakeLists.txt +++ b/src/aliceVision/panorama/CMakeLists.txt @@ -2,7 +2,7 @@ set(panorama_files_headers alphaCompositer.hpp boundingBox.hpp - boundingBoxPanoramaMap.hpp + panoramaMap.hpp compositer.hpp coordinatesMap.hpp distance.hpp @@ -32,7 +32,7 @@ set(panorama_files_sources seams.cpp imageOps.cpp cachedImage.cpp - boundingBoxPanoramaMap.cpp + panoramaMap.cpp ) alicevision_add_library(aliceVision_panorama diff --git a/src/aliceVision/panorama/alphaCompositer.hpp b/src/aliceVision/panorama/alphaCompositer.hpp index 8a5bafdf6a..7a17b9fde1 100644 --- a/src/aliceVision/panorama/alphaCompositer.hpp +++ b/src/aliceVision/panorama/alphaCompositer.hpp @@ -8,8 +8,8 @@ namespace aliceVision class AlphaCompositer : public Compositer { public: - AlphaCompositer(image::TileCacheManager::shared_ptr & cacheManager, size_t outputWidth, size_t outputHeight) - : Compositer(cacheManager, outputWidth, outputHeight) + AlphaCompositer(size_t outputWidth, size_t outputHeight) + : Compositer(outputWidth, outputHeight) { } @@ -18,59 +18,41 @@ class AlphaCompositer : public Compositer const aliceVision::image::Image& inputWeights, int offset_x, int offset_y) { - - aliceVision::image::Image masked(color.Width(), color.Height()); - - BoundingBox panoramaBb; - panoramaBb.left = offset_x; - panoramaBb.top = offset_y; - panoramaBb.width = color.Width(); - panoramaBb.height = color.Height(); - - if (!loopyCachedImageExtract(masked, _panorama, panoramaBb)) + for(int i = 0; i < color.Height(); i++) { - return false; - } - + int y = i + offset_y; + if (y < 0 || y >= _panoramaHeight) continue; - for(size_t i = 0; i < color.Height(); i++) - { - for(size_t j = 0; j < color.Width(); j++) + for (int j = 0; j < color.Width(); j++) { - if(!inputMask(i, j)) + int x = j + offset_x; + if (x < 0 || x >= _panoramaWidth) continue; + + if (!inputMask(i, j)) { continue; } float wc = inputWeights(i, j); - masked(i, j).r() += wc * color(i, j).r(); - masked(i, j).g() += wc * color(i, j).g(); - masked(i, j).b() += wc * color(i, j).b(); - masked(i, j).a() += wc; + _panorama(y, x).r() += wc * color(i, j).r(); + _panorama(y, x).g() += wc * color(i, j).g(); + _panorama(y, x).b() += wc * color(i, j).b(); + _panorama(y, x).a() += wc; } } - BoundingBox inputBb; - inputBb.left = 0; - inputBb.top = 0; - inputBb.width = masked.Width(); - inputBb.height = masked.Height(); - - if (!loopyCachedImageAssign(_panorama, masked, panoramaBb, inputBb)) { - return false; - } - return true; } virtual bool terminate() { - - _panorama.perPixelOperation( - [](image::RGBAfColor c) -> image::RGBAfColor + for (int i = 0; i < _panorama.Height(); i++) + { + for (int j = 0; j < _panorama.Width(); j++) { image::RGBAfColor r; + image::RGBAfColor c = _panorama(i, j); if (c.a() < 1e-6f) { @@ -87,9 +69,9 @@ class AlphaCompositer : public Compositer r.a() = 1.0f; } - return r; + _panorama(i, j) = r; } - ); + } return true; } diff --git a/src/aliceVision/panorama/boundingBox.hpp b/src/aliceVision/panorama/boundingBox.hpp index f7c7fbd1fb..dd30e9fb8e 100644 --- a/src/aliceVision/panorama/boundingBox.hpp +++ b/src/aliceVision/panorama/boundingBox.hpp @@ -199,10 +199,14 @@ struct BoundingBox { BoundingBox b; - b.left = left / factor; - b.top = top / factor; - b.width = width / factor; - b.height = height / factor; + b.left = int(floor(double(left) / double(factor))); + b.top = int(floor(double(top) / double(factor))); + + int right = int(ceil(double(getRight()) / double(factor))); + int bottom = int(ceil(double(getBottom()) / double(factor))); + + b.width = right - b.left + 1; + b.height = bottom - b.top + 1; return b; } diff --git a/src/aliceVision/panorama/boundingBoxPanoramaMap.cpp b/src/aliceVision/panorama/boundingBoxPanoramaMap.cpp deleted file mode 100644 index 53ba6f8ff6..0000000000 --- a/src/aliceVision/panorama/boundingBoxPanoramaMap.cpp +++ /dev/null @@ -1,167 +0,0 @@ -#include "boundingBoxPanoramaMap.hpp" - -#include -#include - -namespace aliceVision -{ - - - -bool BoundingBoxPanoramaMap::append(IndexT index, const BoundingBox & box, int maxScale, int borderSize) -{ - int maxFactor = pow(2, maxScale); - - BoundingBox scaled = box.divide(maxFactor); - - BoundingBox scaledWithBorders = scaled.dilate(borderSize); - - _map[index] = scaledWithBorders.multiply(maxFactor); - - return true; -} - -bool BoundingBoxPanoramaMap::intersect(const BoundingBox & box1, const BoundingBox & box2) const -{ - BoundingBox otherBbox = box2; - BoundingBox otherBboxLoop = box2; - otherBboxLoop.left = otherBbox.left - _panoramaWidth; - BoundingBox otherBboxLoopRight = box2; - otherBboxLoopRight.left = otherBbox.left + _panoramaWidth; - - if (!box1.intersectionWith(otherBbox).isEmpty()) - { - return true; - } - - if (!box1.intersectionWith(otherBboxLoop).isEmpty()) - { - return true; - } - - if (!box1.intersectionWith(otherBboxLoopRight).isEmpty()) - { - return true; - } - - return false; -} - -bool BoundingBoxPanoramaMap::isValid(const std::list & stack) const -{ - if (stack.size() == 0) - { - return false; - } - - if (stack.size() == 1) - { - return true; - } - - auto & item = stack.back(); - IndexT top = *(item.second); - BoundingBox topBb = _map.at(top); - - for (auto it : stack) - { - IndexT current = *(it.second); - if (current == top) continue; - - BoundingBox currentBb = _map.at(current); - - if (intersect(topBb, currentBb)) - { - return false; - } - } - - return true; -} - -bool BoundingBoxPanoramaMap::getBestInitial(std::list & best_items) const -{ - - //For each view, get the list of non overlapping views - //with an id which is superior to them - std::set initialList; - std::map> nonOverlapPerBox; - for (auto it = _map.begin(); it != _map.end(); it++) - { - std::set listNonOverlap; - - initialList.insert(it->first); - - for (auto nextIt = std::next(it); nextIt != _map.end(); nextIt++) - { - if (intersect(it->second, nextIt->second)) - { - continue; - } - - listNonOverlap.insert(nextIt->first); - } - - nonOverlapPerBox[it->first] = listNonOverlap; - } - nonOverlapPerBox[UndefinedIndexT] = initialList; - - size_t best_score = 0; - best_items.clear(); - std::list stack; - stack.push_back(std::make_pair(UndefinedIndexT, nonOverlapPerBox.at(UndefinedIndexT).begin())); - - while (!stack.empty()) - { - if (isValid(stack)) - { - std::list items; - size_t score = 0; - for (auto it : stack) - { - IndexT item; - items.push_back(item); - BoundingBox box = _map.at(item); - score += box.area(); - if (score > best_score) - { - best_items = items; - best_score = score; - } - } - - IndexT id = *stack.back().second; - std::set & addList = nonOverlapPerBox.at(id); - - if (!addList.empty()) - { - stack.push_back(std::make_pair(id, addList.begin())); - continue; - } - } - - while (!stack.empty()) - { - //Iterate over the last list - stackItem & lastItem = stack.back(); - uniqueIndices & list = nonOverlapPerBox.at(lastItem.first); - lastItem.second++; - - //If we reached the end of the list - //Pop back this list - if (stack.back().second == list.end()) - { - stack.pop_back(); - } - else - { - break; - } - } - } - - return true; -} - - -} \ No newline at end of file diff --git a/src/aliceVision/panorama/boundingBoxPanoramaMap.hpp b/src/aliceVision/panorama/boundingBoxPanoramaMap.hpp deleted file mode 100644 index ad2d880a32..0000000000 --- a/src/aliceVision/panorama/boundingBoxPanoramaMap.hpp +++ /dev/null @@ -1,37 +0,0 @@ -#pragma once - -#include "boundingBox.hpp" - -#include - -#include -#include - -namespace aliceVision -{ - -class BoundingBoxPanoramaMap -{ -public: - using uniqueIndices = std::set; - using stackItem = std::pair; -public: - BoundingBoxPanoramaMap(int width, int height) : _panoramaWidth(width), _panoramaHeight(height) - { - - } - - bool append(IndexT index, const BoundingBox & box, int maxScale, int borderSize); - bool getBestInitial(std::list & best_items) const; - -private: - bool intersect(const BoundingBox & box1, const BoundingBox & box2) const; - bool isValid(const std::list & stack) const; - -private: - std::map _map; - int _panoramaWidth; - int _panoramaHeight; -}; - -} \ No newline at end of file diff --git a/src/aliceVision/panorama/compositer.hpp b/src/aliceVision/panorama/compositer.hpp index a017c34d0e..dd377fb058 100644 --- a/src/aliceVision/panorama/compositer.hpp +++ b/src/aliceVision/panorama/compositer.hpp @@ -12,8 +12,7 @@ namespace aliceVision class Compositer { public: - Compositer(image::TileCacheManager::shared_ptr & cacheManager, int width, int height) : - _cacheManager(cacheManager), + Compositer(int width, int height) : _panoramaWidth(width), _panoramaHeight(height) { @@ -25,65 +24,34 @@ class Compositer const aliceVision::image::Image& inputWeights, int offset_x, int offset_y) { - aliceVision::image::Image masked(color.Width(), color.Height()); - - BoundingBox panoramaBb; - panoramaBb.left = offset_x; - panoramaBb.top = offset_y; - panoramaBb.width = color.Width(); - panoramaBb.height = color.Height(); - - if (!loopyCachedImageExtract(masked, _panorama, panoramaBb)) + for(int i = 0; i < color.Height(); i++) { - return false; - } - + int y = i + offset_y; + if (y < 0 || y >= _panoramaHeight) continue; - for(size_t i = 0; i < color.Height(); i++) - { - for(size_t j = 0; j < color.Width(); j++) + for (int j = 0; j < color.Width(); j++) { - if(!inputMask(i, j)) + int x = j + offset_x; + if (x < 0 || x >= _panoramaWidth) continue; + + if (!inputMask(i, j)) { continue; } - masked(i, j).r() = color(i, j).r(); - masked(i, j).g() = color(i, j).g(); - masked(i, j).b() = color(i, j).b(); - masked(i, j).a() = 1.0f; + _panorama(y, x).r() = color(i, j).r(); + _panorama(y, x).g() = color(i, j).g(); + _panorama(y, x).b() = color(i, j).b(); + _panorama(y, x).a() = 1.0f; } } - BoundingBox inputBb; - inputBb.left = 0; - inputBb.top = 0; - inputBb.width = color.Width(); - inputBb.height = color.Height(); - - if (!loopyCachedImageAssign(_panorama, masked, panoramaBb, inputBb)) { - return false; - } - return true; } virtual bool initialize() { - if(!_panorama.createImage(_cacheManager, _panoramaWidth, _panoramaHeight)) - { - return false; - } - - if(!_panorama.perPixelOperation( - [](image::RGBAfColor ) -> image::RGBAfColor - { - return image::RGBAfColor(0.0f, 0.0f, 0.0f, 0.0f); - }) - ) - { - return false; - } + _panorama = image::Image(_panoramaWidth, _panoramaHeight, true, image::RGBAfColor(0.0f, 0.0f, 0.0f, 0.0f)); return true; } @@ -94,25 +62,28 @@ class Compositer { if (storageDataType == image::EStorageDataType::HalfFinite) { - _panorama.perPixelOperation([](const image::RGBAfColor & c) + for (int i = 0; i < _panorama.Height(); i++) { - image::RGBAfColor ret; + for (int j = 0; j < _panorama.Width(); j++) + { + image::RGBAfColor ret; + image::RGBAfColor c = _panorama(i, j); - const float limit = float(HALF_MAX); - - ret.r() = clamp(c.r(), -limit, limit); - ret.g() = clamp(c.g(), -limit, limit); - ret.b() = clamp(c.b(), -limit, limit); - ret.a() = c.a(); + const float limit = float(HALF_MAX); + + ret.r() = clamp(c.r(), -limit, limit); + ret.g() = clamp(c.g(), -limit, limit); + ret.b() = clamp(c.b(), -limit, limit); + ret.a() = c.a(); - return ret; - }); + _panorama(i, j) = ret; + } + } } - if(!_panorama.writeImage(path, storageDataType)) - { - return false; - } + oiio::ParamValueList metadata; + metadata.push_back(oiio::ParamValue("AliceVision:storageDataType", EStorageDataType_enumToString(storageDataType))); + image::writeImage(path, _panorama, image::EImageColorSpace::LINEAR, metadata); return true; } @@ -127,19 +98,9 @@ class Compositer return 0; } - bool drawBorders(const aliceVision::image::Image& mask, size_t offsetX, size_t offsetY) - { - return ::aliceVision::drawBorders(_panorama, mask, offsetX, offsetY); - } - - bool drawSeams(CachedImage& label) - { - return ::aliceVision::drawSeams(_panorama, label); - } - protected: image::TileCacheManager::shared_ptr _cacheManager; - CachedImage _panorama; + image::Image _panorama; int _panoramaWidth; int _panoramaHeight; }; diff --git a/src/aliceVision/panorama/graphcut.hpp b/src/aliceVision/panorama/graphcut.hpp index 685e6023ad..924f0c79ac 100644 --- a/src/aliceVision/panorama/graphcut.hpp +++ b/src/aliceVision/panorama/graphcut.hpp @@ -519,14 +519,12 @@ class GraphcutSeams //Build the input - ALICEVISION_LOG_INFO("build input"); image::Image graphCutInput(localBbox.width, localBbox.height, true); if (!createInputOverlappingObservations(graphCutInput, localBbox, computeExpansion)) { return false; } - ALICEVISION_LOG_INFO("fix input"); // Fix upscaling induced bad labeling if (!fixUpscaling(localLabels, graphCutInput)) { @@ -534,7 +532,6 @@ class GraphcutSeams } // Backup update for upscaling - ALICEVISION_LOG_INFO("save input"); BoundingBox inputBb = localBbox; inputBb.left = 0; inputBb.top = 0; diff --git a/src/aliceVision/panorama/imageOps.cpp b/src/aliceVision/panorama/imageOps.cpp index f37102621c..83b6b5475d 100644 --- a/src/aliceVision/panorama/imageOps.cpp +++ b/src/aliceVision/panorama/imageOps.cpp @@ -5,17 +5,18 @@ namespace aliceVision { -void removeNegativeValues(CachedImage & img) +void removeNegativeValues(image::Image & img) { - img.perPixelOperation( - [](const image::RGBfColor & c) -> image::RGBfColor + for (int i = 0; i < img.Height(); i++) + { + for (int j = 0; j < img.Width(); j++) { image::RGBfColor rpix; - image::RGBfColor ret = c; + image::RGBfColor ret = img(i, j); - rpix.r() = std::exp(c.r()); - rpix.g() = std::exp(c.g()); - rpix.b() = std::exp(c.b()); + rpix.r() = std::exp(ret.r()); + rpix.g() = std::exp(ret.g()); + rpix.b() = std::exp(ret.b()); if (rpix.r() < 0.0) { @@ -32,9 +33,9 @@ void removeNegativeValues(CachedImage & img) ret.b() = 0.0; } - return ret; + img(i, j) = ret; } - ); + } } } // namespace aliceVision \ No newline at end of file diff --git a/src/aliceVision/panorama/imageOps.hpp b/src/aliceVision/panorama/imageOps.hpp index c696fc7421..6f3ae5ee88 100644 --- a/src/aliceVision/panorama/imageOps.hpp +++ b/src/aliceVision/panorama/imageOps.hpp @@ -163,7 +163,7 @@ bool addition(aliceVision::image::Image& AplusB, const aliceVision::image::Im return true; } -void removeNegativeValues(CachedImage& img); +void removeNegativeValues(image::Image& img); template bool loopyCachedImageAssign(CachedImage & output, const aliceVision::image::Image & input, const BoundingBox & assignedOutputBb, const BoundingBox & assignedInputBb) @@ -279,9 +279,9 @@ bool loopyCachedImageExtract(aliceVision::image::Image & output, CachedImage< template bool makeImagePyramidCompatible(image::Image& output, - size_t& outOffsetX, size_t& outOffsetY, + int & outOffsetX, int & outOffsetY, const image::Image& input, - size_t offsetX, size_t offsetY, + int offsetX, int offsetY, size_t borderSize, size_t num_levels) { @@ -301,12 +301,12 @@ bool makeImagePyramidCompatible(image::Image& output, double correctedLowOffsetY = floor(lowOffsetY); /*Add some borders on the top and left to make sure mask can be smoothed*/ - correctedLowOffsetX = std::max(0.0, correctedLowOffsetX - double(borderSize)); - correctedLowOffsetY = std::max(0.0, correctedLowOffsetY - double(borderSize)); + correctedLowOffsetX = correctedLowOffsetX - double(borderSize); + correctedLowOffsetY = correctedLowOffsetY - double(borderSize); /*Compute offset at largest level*/ - outOffsetX = size_t(correctedLowOffsetX / maxScale); - outOffsetY = size_t(correctedLowOffsetY / maxScale); + outOffsetX = int(correctedLowOffsetX / maxScale); + outOffsetY = int(correctedLowOffsetY / maxScale); /*Compute difference*/ double doffsetX = double(offsetX) - double(outOffsetX); diff --git a/src/aliceVision/panorama/laplacianCompositer.hpp b/src/aliceVision/panorama/laplacianCompositer.hpp index fa198b1c70..365248bb9c 100644 --- a/src/aliceVision/panorama/laplacianCompositer.hpp +++ b/src/aliceVision/panorama/laplacianCompositer.hpp @@ -10,8 +10,8 @@ namespace aliceVision class LaplacianCompositer : public Compositer { public: - LaplacianCompositer(image::TileCacheManager::shared_ptr & cacheManager, size_t outputWidth, size_t outputHeight) - : Compositer(cacheManager, outputWidth, outputHeight) + LaplacianCompositer(size_t outputWidth, size_t outputHeight) + : Compositer(outputWidth, outputHeight) , _pyramidPanorama(outputWidth, outputHeight, 1) , _bands(1) { @@ -24,7 +24,7 @@ class LaplacianCompositer : public Compositer return false; } - return _pyramidPanorama.initialize(_cacheManager); + return _pyramidPanorama.initialize(); } virtual size_t getOptimalScale(int width, int height) const @@ -43,7 +43,7 @@ class LaplacianCompositer : public Compositer size_t optimal_scale = size_t(floor(std::log2(double(minsize) / gaussianFilterSize))); - return (optimal_scale - 1/*Security*/); + return 5;//(optimal_scale - 1/*Security*/); } virtual int getBorderSize() const @@ -54,7 +54,7 @@ class LaplacianCompositer : public Compositer virtual bool append(const aliceVision::image::Image& color, const aliceVision::image::Image& inputMask, const aliceVision::image::Image& inputWeights, - int offsetX, int offsetY) + int offsetX, int offsetY, size_t optimalScale) { size_t optimalScale = getOptimalScale(color.Width(), color.Height()); size_t optimalLevelsCount = optimalScale + 1; @@ -70,7 +70,7 @@ class LaplacianCompositer : public Compositer if(optimalLevelsCount > _bands && _bands == 1) { _bands = optimalLevelsCount; - if (!_pyramidPanorama.augment(_cacheManager, _bands)) + if (!_pyramidPanorama.augment(_bands)) { return false; } @@ -78,11 +78,12 @@ class LaplacianCompositer : public Compositer // Make sure input is compatible with pyramid processing // See comments inside function for details - size_t newOffsetX, newOffsetY; + int newOffsetX, newOffsetY; aliceVision::image::Image colorPot; aliceVision::image::Image maskPot; aliceVision::image::Image weightsPot; + makeImagePyramidCompatible(colorPot, newOffsetX, newOffsetY, color, offsetX, offsetY, getBorderSize(), _bands); makeImagePyramidCompatible(maskPot, newOffsetX, newOffsetY, inputMask, offsetX, offsetY, getBorderSize(), _bands); makeImagePyramidCompatible(weightsPot, newOffsetX, newOffsetY, inputWeights, offsetX, offsetY, getBorderSize(), _bands); @@ -95,7 +96,7 @@ class LaplacianCompositer : public Compositer return false; } - /*To log space for hdr*/ + // To log space for hdr for(int i = 0; i < feathered.Height(); i++) { for(int j = 0; j < feathered.Width(); j++) @@ -106,7 +107,7 @@ class LaplacianCompositer : public Compositer } } - /* Convert mask to alpha layer */ + // Convert mask to alpha layer image::Image maskFloat(maskPot.Width(), maskPot.Height()); for(int i = 0; i < maskPot.Height(); i++) { @@ -133,24 +134,26 @@ class LaplacianCompositer : public Compositer virtual bool terminate() { - if (!_pyramidPanorama.rebuild(_panorama)) { return false; } - _panorama.perPixelOperation( - [](const image::RGBAfColor & a) -> image::RGBAfColor { + for (int i = 0; i < _panorama.Height(); i++) + { + for (int j = 0; j < _panorama.Width(); j++) + { + image::RGBAfColor c = _panorama(i, j); image::RGBAfColor out; - out.r() = std::exp(a.r()); - out.g() = std::exp(a.g()); - out.b() = std::exp(a.b()); - out.a() = a.a(); + out.r() = std::exp(c.r()); + out.g() = std::exp(c.g()); + out.b() = std::exp(c.b()); + out.a() = c.a(); - return out; + _panorama(i, j) = out; } - ); + } return true; } diff --git a/src/aliceVision/panorama/laplacianPyramid.cpp b/src/aliceVision/panorama/laplacianPyramid.cpp index 32e0b40f50..ebd33738aa 100644 --- a/src/aliceVision/panorama/laplacianPyramid.cpp +++ b/src/aliceVision/panorama/laplacianPyramid.cpp @@ -15,7 +15,7 @@ _maxLevels(max_levels) } -bool LaplacianPyramid::initialize(image::TileCacheManager::shared_ptr & cacheManager) +bool LaplacianPyramid::initialize() { size_t width = _baseWidth; size_t height = _baseHeight; @@ -26,9 +26,6 @@ bool LaplacianPyramid::initialize(image::TileCacheManager::shared_ptr & cacheMan /*Prepare pyramid*/ for(int lvl = 0; lvl < _maxLevels; lvl++) { - CachedImage color; - CachedImage weights; - //If the level width is odd, it is a problem. //Let update this level to an even size. we'll manage it later //Store the real size to know we updated it @@ -38,25 +35,8 @@ bool LaplacianPyramid::initialize(image::TileCacheManager::shared_ptr & cacheMan width++; } - if(!color.createImage(cacheManager, width, height)) - { - return false; - } - - if(!weights.createImage(cacheManager, width, height)) - { - return false; - } - - if(!color.fill(image::RGBfColor(0.0f, 0.0f, 0.0f))) - { - return false; - } - - if(!weights.fill(0.0f)) - { - return false; - } + image::Image color(width, height, true, image::RGBfColor(0.0f, 0.0f, 0.0f)); + image::Image weights(width, height, true, 0.0f); _levels.push_back(color); _weights.push_back(weights); @@ -68,7 +48,7 @@ bool LaplacianPyramid::initialize(image::TileCacheManager::shared_ptr & cacheMan return true; } -bool LaplacianPyramid::augment(image::TileCacheManager::shared_ptr & cacheManager, size_t newMaxLevels) +bool LaplacianPyramid::augment(size_t newMaxLevels) { ALICEVISION_LOG_INFO("augment number of levels to " << newMaxLevels); @@ -84,11 +64,8 @@ bool LaplacianPyramid::augment(image::TileCacheManager::shared_ptr & cacheManage // Augment the number of levels in the pyramid for (int level = oldMaxLevels; level < newMaxLevels; level++) { - CachedImage pyramidImage; - CachedImage pyramidWeights; - - int width = _levels[_levels.size() - 1].getWidth(); - int height = _levels[_levels.size() - 1].getHeight(); + int width = _levels[_levels.size() - 1].Width(); + int height = _levels[_levels.size() - 1].Height(); int nextWidth = width / 2; int nextHeight = int(ceil(float(height) / 2.0f)); @@ -102,18 +79,8 @@ bool LaplacianPyramid::augment(image::TileCacheManager::shared_ptr & cacheManage nextWidth++; } - if(!pyramidImage.createImage(cacheManager, nextWidth, nextHeight)) - { - return false; - } - - if(!pyramidWeights.createImage(cacheManager, nextWidth, nextHeight)) - { - return false; - } - - pyramidImage.fill(image::RGBfColor(0.0f)); - pyramidWeights.fill(0.0f); + image::Image pyramidImage(nextWidth, nextHeight, true, image::RGBfColor(0.0f)); + image::Image pyramidWeights(nextWidth, nextHeight, true, 0.0f); _levels.push_back(pyramidImage); _weights.push_back(pyramidWeights); @@ -139,7 +106,7 @@ bool LaplacianPyramid::augment(image::TileCacheManager::shared_ptr & cacheManage bool LaplacianPyramid::apply(const aliceVision::image::Image& source, const aliceVision::image::Image& mask, const aliceVision::image::Image& weights, - size_t initialLevel, size_t offsetX, size_t offsetY) + size_t initialLevel, int offsetX, int offsetY) { //We assume the input source has been feathered //and resized to be a simili power of 2 for the needed scales. @@ -155,7 +122,7 @@ bool LaplacianPyramid::apply(const aliceVision::image::Image& for(int l = initialLevel; l < _levels.size() - 1; l++) { - CachedImage & img = _levels[l]; + image::Image & img = _levels[l]; BoundingBox inputBbox; inputBbox.left = offsetX; @@ -163,71 +130,12 @@ bool LaplacianPyramid::apply(const aliceVision::image::Image& inputBbox.width = width; inputBbox.height = height; - //If the destination image size is odd, - //It is a big problem - bool specialLoop = false; - int loopPosition = std::numeric_limits::max(); - int realLevelWidth = _realWidths[l]; - int offset = 0; - - if (img.getWidth() != realLevelWidth) - { - //Do we cross the loop ? - if (inputBbox.getRight() > img.getWidth()) - { - specialLoop = true; - loopPosition = realLevelWidth - inputBbox.left; - offset = 2; - } - } - - //Create aligned images if necessary - if (specialLoop) - { - image::Image alignedColor(width + offset, height); - image::Image alignedWeights(width + offset, height); - image::Image alignedMask(width + offset, height); - - for(int i = 0; i < currentColor.Height(); i++) - { - int dj = 0; - - for(int j = 0; j < currentColor.Width(); j++) - { - if (j == loopPosition) - { - dj++; - } - - alignedMask(i, dj) = currentMask(i, j); - alignedColor(i, dj) = currentColor(i, j); - alignedWeights(i, dj) = currentWeights(i, j); - - dj++; - } - - if (specialLoop) - { - alignedColor(i, loopPosition) = alignedColor(i, loopPosition + 1); - alignedMask(i, loopPosition) = alignedMask(i, loopPosition + 1); - alignedWeights(i, loopPosition) = alignedWeights(i, loopPosition + 1); - alignedMask(i, width + 1) = 0; - alignedWeights(i, width + 1) = 0; - } - } - - currentColor = alignedColor; - currentWeights = alignedWeights; - currentMask = alignedMask; - } - + image::Image bufMasked(width, height); + image::Image buf(width, height); + image::Image buf2(width, height); + image::Image bufFloat(width, height); - image::Image bufMasked(width + offset, height); - image::Image buf(width + offset, height); - image::Image buf2(width + offset, height); - image::Image bufFloat(width + offset, height); - - /*Apply mask to content before convolution*/ + // Apply mask to content before convolution for(int i = 0; i < height; i++) { for(int j = 0; j < width; j++) @@ -365,69 +273,39 @@ bool LaplacianPyramid::apply(const aliceVision::image::Image& iinfo.weights = currentWeights; _inputInfos.push_back(iinfo); - /*if (!merge(currentColor, currentWeights, _levels.size() - 1, offsetX, offsetY)) - { - return false; - }*/ return true; } bool LaplacianPyramid::merge(const aliceVision::image::Image& oimg, - const aliceVision::image::Image& oweight, size_t level, size_t offsetX, - size_t offsetY) + const aliceVision::image::Image& oweight, + size_t level, int offsetX, int offsetY) { - CachedImage & img = _levels[level]; - CachedImage & weight = _weights[level]; - - aliceVision::image::Image extractedColor(oimg.Width(), oimg.Height()); - aliceVision::image::Image extractedWeight(oimg.Width(), oimg.Height()); - - BoundingBox extractBb; - extractBb.left = offsetX; - extractBb.top = offsetY; - extractBb.width = oimg.Width(); - extractBb.height = oimg.Height(); - extractBb.clampBottom(img.getHeight() - 1); - - - BoundingBox inputBb = extractBb; - inputBb.left = 0; - inputBb.top = 0; - - if (!loopyCachedImageExtract(extractedColor, img, extractBb)) - { - return false; - } + image::Image & img = _levels[level]; + image::Image & weight = _weights[level]; - if (!loopyCachedImageExtract(extractedWeight, weight, extractBb)) - { - return false; - } - - for(int y = 0; y < inputBb.height; y++) + for(int i = 0; i < oimg.Height(); i++) { - for(int x = 0; x < inputBb.width; x++) + int y = i + offsetY; + if (y < 0 || y >= img.Height()) continue; + + for(int j = 0; j < oimg.Width(); j++) { - extractedColor(y, x).r() += oimg(y, x).r() * oweight(y, x); - extractedColor(y, x).g() += oimg(y, x).g() * oweight(y, x); - extractedColor(y, x).b() += oimg(y, x).b() * oweight(y, x); - extractedWeight(y, x) += oweight(y, x); + int x = j + offsetX; + if (x < 0 || x >= img.Width()) continue; + + img(y, x).r() += oimg(i, j).r() * oweight(i, j); + img(y, x).g() += oimg(i, j).g() * oweight(i, j); + img(y, x).b() += oimg(i, j).b() * oweight(i, j); + weight(y, x) += oweight(i, j); } } - if (!loopyCachedImageAssign(img, extractedColor, extractBb, inputBb)) { - return false; - } - - if (!loopyCachedImageAssign(weight, extractedWeight, extractBb, inputBb)) { - return false; - } return true; } -bool LaplacianPyramid::rebuild(CachedImage& output) +bool LaplacianPyramid::rebuild(image::Image& output) { for (InputInfo & iinfo : _inputInfos) { @@ -440,12 +318,20 @@ bool LaplacianPyramid::rebuild(CachedImage& output) // We first want to compute the final pixels mean for(int l = 0; l < _levels.size(); l++) { - _levels[l].perPixelOperation(_weights[l], - [](const image::RGBfColor & c, const float & w) -> image::RGBfColor + image::Image & level = _levels[l]; + image::Image & weight = _weights[l]; + + for (int i = 0; i < level.Height(); i++) + { + for (int j = 0; j < level.Width(); j++) { + float w = weight(i, j); + image::RGBfColor c = level(i, j); + if (w < 1e-6) { - return image::RGBfColor(0.0f, 0.0f, 0.0f); + level(i, j) = image::RGBfColor(0.0f, 0.0f, 0.0f); + continue; } image::RGBfColor r; @@ -454,158 +340,64 @@ bool LaplacianPyramid::rebuild(CachedImage& output) r.g() = c.g() / w; r.b() = c.b() / w; - return r; + level(i, j) = r; } - ); + } } removeNegativeValues(_levels[_levels.size() - 1]); for(int l = _levels.size() - 2; l >= 0; l--) { - const size_t processingSize = 512; - const size_t borderSize = 5; - int halfLevel = l + 1; int currentLevel = l; - for (int py = 0; py < _levels[halfLevel].getHeight(); py += processingSize) - { - for (int px = 0; px < _levels[halfLevel].getWidth(); px += processingSize) - { - //Estimate the Bounding box - //Make sure we are not processing outside of the image bounding box - BoundingBox extractedBb; - extractedBb.left = px; - extractedBb.top = py; - extractedBb.width = processingSize; - extractedBb.height = processingSize; - extractedBb.clampLeft(); - extractedBb.clampTop(); - extractedBb.clampRight(_levels[halfLevel].getWidth() - 1); - extractedBb.clampBottom(_levels[halfLevel].getHeight() - 1); - - BoundingBox doubleBb = extractedBb.doubleSize(); - doubleBb.clampLeft(); - doubleBb.clampTop(); - doubleBb.clampRight(_levels[currentLevel].getWidth() - 1); - doubleBb.clampBottom(_levels[currentLevel].getHeight() - 1); - - //Add borders to this bounding box for filtering - BoundingBox dilatedBb = extractedBb.dilate(borderSize); - dilatedBb.clampTop(); - dilatedBb.clampBottom(_levels[halfLevel].getHeight() - 1); - - //If the box has a negative left border, - //it is equivalent (with the loop, to shifting at the end) - int leftBorder = extractedBb.left - dilatedBb.left; - int topBorder = extractedBb.top - dilatedBb.top; - if (dilatedBb.left < 0) - { - dilatedBb.left = _levels[halfLevel].getWidth() + dilatedBb.left; - } - - aliceVision::image::Image extracted(dilatedBb.width, dilatedBb.height); - if (!loopyCachedImageExtract(extracted, _levels[halfLevel], dilatedBb)) - { - return false; - } + aliceVision::image::Image buf(_levels[currentLevel].Width(), _levels[currentLevel].Height()); + aliceVision::image::Image buf2(_levels[currentLevel].Width(), _levels[currentLevel].Height()); - aliceVision::image::Image extractedNext(doubleBb.width, doubleBb.height); - if (!loopyCachedImageExtract(extractedNext, _levels[currentLevel], doubleBb)) - { - return false; - } - - aliceVision::image::Image buf(dilatedBb.width * 2, dilatedBb.height * 2); - aliceVision::image::Image buf2(dilatedBb.width * 2, dilatedBb.height * 2); - - if (!upscale(buf, extracted)) - { - return false; - } - - - if (!convolveGaussian5x5(buf2, buf, false)) - { - return false; - } + if (!upscale(buf, _levels[halfLevel])) + { + return false; + } - for(int y = 0; y < buf2.Height(); y++) - { - for(int x = 0; x < buf2.Width(); x++) - { - buf2(y, x) *= 4.0f; - } - } - - int shiftY = topBorder * 2; - int shiftX = leftBorder * 2; - for (int y = 0; y < doubleBb.height; y++) - { - for (int x = 0; x < doubleBb.width; x++) - { - extractedNext(y, x) += buf2(y + shiftY, x + shiftX); - } - } + if (!convolveGaussian5x5(buf2, buf, false)) + { + return false; + } - BoundingBox inputAssigmentBb = doubleBb; - inputAssigmentBb.left = 0; - inputAssigmentBb.top = 0; - if (!loopyCachedImageAssign(_levels[currentLevel], extractedNext, doubleBb, inputAssigmentBb)) - { - std::cout << "failed assign" << std::endl; - return false; - } + for(int y = 0; y < buf2.Height(); y++) + { + for(int x = 0; x < buf2.Width(); x++) + { + buf2(y, x) *= 4.0f; } } + + if (!addition(_levels[currentLevel], _levels[currentLevel], buf2)) + { + return false; + } removeNegativeValues(_levels[currentLevel]); } - - for(int i = 0; i < output.getTiles().size(); i++) + + image::Image & level = _levels[0]; + image::Image & weight = _weights[0]; + for(int i = 0; i < output.Height(); i++) { - - std::vector & rowOutput = output.getTiles()[i]; - std::vector & rowInput = _levels[0].getTiles()[i]; - std::vector & rowWeight = _weights[0].getTiles()[i]; - - for(int j = 0; j < rowOutput.size(); j++) + for(int j = 0; j < output.Width(); j++) { + output(i, j).r() = level(i, j).r(); + output(i, j).g() = level(i, j).g(); + output(i, j).b() = level(i, j).b(); - if(!rowOutput[j]->acquire()) - { - return false; - } - - if(!rowInput[j]->acquire()) + if (weight(i, j) < 1e-6) { - return false; + output(i, j).a() = 0.0f; } - - if(!rowWeight[j]->acquire()) + else { - return false; - } - - image::RGBAfColor* ptrOutput = (image::RGBAfColor *)rowOutput[j]->getDataPointer(); - image::RGBfColor* ptrInput = (image::RGBfColor *)rowInput[j]->getDataPointer(); - float* ptrWeight = (float *)rowWeight[j]->getDataPointer(); - - for (int k = 0; k < output.getTileSize() * output.getTileSize(); k++) - { - ptrOutput[k].r() = ptrInput[k].r(); - ptrOutput[k].g() = ptrInput[k].g(); - ptrOutput[k].b() = ptrInput[k].b(); - - if(ptrWeight[k] < 1e-6) - { - ptrOutput[k].a() = 0.0f; - } - else - { - ptrOutput[k].a() = 1.0f; - } + output(i, j).a() = 1.0f; } } } diff --git a/src/aliceVision/panorama/laplacianPyramid.hpp b/src/aliceVision/panorama/laplacianPyramid.hpp index 7185eb6ba0..ad51d0dfe4 100644 --- a/src/aliceVision/panorama/laplacianPyramid.hpp +++ b/src/aliceVision/panorama/laplacianPyramid.hpp @@ -2,7 +2,7 @@ #include "imageOps.hpp" -#include "cachedImage.hpp" +#include namespace aliceVision { @@ -22,26 +22,28 @@ class LaplacianPyramid public: LaplacianPyramid(size_t base_width, size_t base_height, size_t max_levels); - bool initialize(image::TileCacheManager::shared_ptr & cacheManager); + bool initialize(); - bool augment(image::TileCacheManager::shared_ptr & cacheManager, size_t new_max_levels); + bool augment(size_t new_max_levels); bool apply(const aliceVision::image::Image& source, - const aliceVision::image::Image& mask, const aliceVision::image::Image& weights, - size_t initialLevel, size_t offset_x, size_t offset_y); + const aliceVision::image::Image& mask, + const aliceVision::image::Image& weights, + size_t initialLevel, int offset_x, int offset_y); - bool merge(const aliceVision::image::Image& oimg, const aliceVision::image::Image& oweight, - size_t level, size_t offset_x, size_t offset_y); + bool merge(const aliceVision::image::Image& oimg, + const aliceVision::image::Image& oweight, + size_t level, int offset_x, int offset_y); - bool rebuild(CachedImage& output); + bool rebuild(image::Image& output); private: int _baseWidth; int _baseHeight; int _maxLevels; - std::vector> _levels; - std::vector> _weights; + std::vector> _levels; + std::vector> _weights; std::vector _realWidths; std::vector _inputInfos; }; diff --git a/src/aliceVision/panorama/panoramaMap.cpp b/src/aliceVision/panorama/panoramaMap.cpp new file mode 100644 index 0000000000..8d02e2ff28 --- /dev/null +++ b/src/aliceVision/panorama/panoramaMap.cpp @@ -0,0 +1,112 @@ +#include "panoramaMap.hpp" + +#include +#include + +namespace aliceVision +{ + +bool PanoramaMap::append(IndexT index, const BoundingBox & box) +{ + int maxFactor = pow(2, _scale); + + BoundingBox scaled = box.divide(maxFactor); + + BoundingBox scaledWithBorders = scaled.dilate(_borderSize); + + _map[index] = scaledWithBorders.multiply(maxFactor); + + return true; +} + +bool PanoramaMap::intersect(const BoundingBox & box1, const BoundingBox & box2) const +{ + BoundingBox otherBbox = box2; + BoundingBox otherBboxLoop = box2; + otherBboxLoop.left = otherBbox.left - _panoramaWidth; + BoundingBox otherBboxLoopRight = box2; + otherBboxLoopRight.left = otherBbox.left + _panoramaWidth; + + if (!box1.intersectionWith(otherBbox).isEmpty()) + { + /*BoundingBox sbox1 = box1.divide(_scale); + BoundingBox sotherBbox = otherBbox.divide(_scale); + BoundingBox bb = sbox1.intersectionWith(sotherBbox); + BoundingBox bbb = bb.multiply(_scale); + + std::cout << bbb << std::endl; + std::cout << "(" << box2 << ")" << std::endl;*/ + + return true; + } + + if (!box1.intersectionWith(otherBbox).isEmpty()) + { + /*BoundingBox sbox1 = box1.divide(_scale); + BoundingBox sotherBbox = otherBboxLoop.divide(_scale); + BoundingBox bb = sbox1.intersectionWith(sotherBbox); + BoundingBox bbb = bb.multiply(_scale); + + std::cout << bbb << std::endl; + std::cout << "(" << box2 << ")" << std::endl;*/ + + return true; + } + + if (!box1.intersectionWith(otherBboxLoopRight).isEmpty()) + { + /*BoundingBox sbox1 = box1.divide(_scale); + BoundingBox sotherBbox = otherBboxLoopRight.divide(_scale); + BoundingBox bb = sbox1.intersectionWith(sotherBbox); + BoundingBox bbb = bb.multiply(_scale); + + std::cout << bbb << std::endl; + std::cout << "(" << box2 << ")" << std::endl;*/ + + return true; + } + + return false; +} + +bool PanoramaMap::getOverlaps(std::list & overlaps, IndexT reference) +{ + if (_map.find(reference) == _map.end()) + { + return false; + } + + BoundingBox bbref = _map[reference]; + + for (auto it : _map) + { + if (it.first == reference) + { + continue; + } + + if (intersect(bbref, it.second)) + { + overlaps.push_back(it.first); + } + } + + return true; +} + +bool PanoramaMap::getOverlaps(std::list & overlaps, BoundingBox bbref) +{ + + + for (auto it : _map) + { + if (intersect(bbref, it.second)) + { + overlaps.push_back(it.first); + } + } + + return true; +} + +} \ No newline at end of file diff --git a/src/aliceVision/panorama/panoramaMap.hpp b/src/aliceVision/panorama/panoramaMap.hpp new file mode 100644 index 0000000000..a1c613c566 --- /dev/null +++ b/src/aliceVision/panorama/panoramaMap.hpp @@ -0,0 +1,63 @@ +#pragma once + +#include "boundingBox.hpp" + +#include + +#include +#include + +namespace aliceVision +{ + +class PanoramaMap +{ +public: + PanoramaMap(int width, int height, int scale, int borderSize) : + _panoramaWidth(width), + _panoramaHeight(height), + _scale(scale), + _borderSize(borderSize) + { + } + + bool append(IndexT index, const BoundingBox & box); + + bool getOverlaps(std::list & overlaps, IndexT reference); + bool getOverlaps(std::list & overlaps, BoundingBox bbref); + + size_t getWidth() const + { + return _panoramaWidth; + } + + size_t geHeight() const + { + return _panoramaHeight; + } + + bool getEnhancedBoundingBox(BoundingBox & bb, const IndexT & id) + { + if (_map.find(id) == _map.end()) + { + return false; + } + + bb = _map[id]; + + return true; + } + +private: + bool intersect(const BoundingBox & box1, const BoundingBox & box2) const; + +private: + std::map _map; + + int _panoramaWidth; + int _panoramaHeight; + int _scale; + int _borderSize; +}; + +} \ No newline at end of file diff --git a/src/aliceVision/panorama/remapBbox.cpp b/src/aliceVision/panorama/remapBbox.cpp index 254152189a..42a25ac092 100644 --- a/src/aliceVision/panorama/remapBbox.cpp +++ b/src/aliceVision/panorama/remapBbox.cpp @@ -1,5 +1,3 @@ -#pragma once - #include "remapBbox.hpp" #include "sphericalMapping.hpp" diff --git a/src/aliceVision/panorama/seams.cpp b/src/aliceVision/panorama/seams.cpp index cacd595d1f..7109bf3334 100644 --- a/src/aliceVision/panorama/seams.cpp +++ b/src/aliceVision/panorama/seams.cpp @@ -270,91 +270,38 @@ bool drawSeams(CachedImage& inout, CachedImage& label return true; } -bool WTASeams::initialize(image::TileCacheManager::shared_ptr & cacheManager) -{ - if(!_weights.createImage(cacheManager, _panoramaWidth, _panoramaHeight)) - { - return false; - } - - if(!_weights.fill(0.0f)) - { - return false; - } - - if(!_labels.createImage(cacheManager, _panoramaWidth, _panoramaHeight)) - { - return false; - } - - if(!_labels.fill(UndefinedIndexT)) - { - return false; - } - - return true; -} - bool WTASeams::append(const aliceVision::image::Image& inputMask, - const aliceVision::image::Image& inputWeights, IndexT currentIndex, size_t offset_x, - size_t offset_y) + const aliceVision::image::Image& inputWeights, + IndexT currentIndex, size_t offset_x, size_t offset_y) { - if( inputMask.size() != inputWeights.size()) + if (inputMask.size() != inputWeights.size()) { return false; - } - - aliceVision::image::Image weights(inputMask.Width(), inputMask.Height()); - aliceVision::image::Image labels(inputMask.Width(), inputMask.Height()); - - BoundingBox globalBb; - globalBb.left = offset_x; - globalBb.top = offset_y; - globalBb.width = inputMask.Width(); - globalBb.height = inputMask.Height(); - - if (!loopyCachedImageExtract(weights, _weights, globalBb)) - { - return false; - } + } - if (!loopyCachedImageExtract(labels, _labels, globalBb)) + for (size_t i = 0; i < inputWeights.Height(); i++) { - return false; - } - + int y = i + offset_y; + if (y < 0 || y >= _panoramaHeight) continue; - for(size_t i = 0; i < weights.Height(); i++) - { - for(size_t j = 0; j < weights.Width(); j++) + for (size_t j = 0; j < inputWeights.Width(); j++) { - if(!inputMask(i, j)) + int x = j + offset_x; + if (x < 0 || x >= _panoramaWidth) continue; + + if (!inputMask(i, j)) { continue; } - if (inputWeights(i, j) > weights(i, j)) + if (inputWeights(i, j) > _weights(y, x)) { - labels(i, j) = currentIndex; - weights(i, j) = inputWeights(i, j); + _labels(y, x) = currentIndex; + _weights(y, x) = inputWeights(i, j); } } } - BoundingBox inputBb; - inputBb.left = 0; - inputBb.top = 0; - inputBb.width = labels.Width(); - inputBb.height = labels.Height(); - - if (!loopyCachedImageAssign(_weights, weights, globalBb, inputBb)) { - return false; - } - - if (!loopyCachedImageAssign(_labels, labels, globalBb, inputBb)) { - return false; - } - return true; } @@ -432,7 +379,7 @@ bool HierarchicalGraphcutSeams::append(const aliceVision::image::Image potImage; aliceVision::image::Image potMask; makeImagePyramidCompatible(potImage, newOffsetX, newOffsetY, input, offsetX, offsetY, 2, _countLevels); @@ -681,33 +628,25 @@ bool HierarchicalGraphcutSeams::initialize() return true; } -bool getMaskFromLabels(aliceVision::image::Image & mask, CachedImage & labels, IndexT index, size_t offset_x, size_t offset_y) { - - image::Image extractedLabels(mask.Width(), mask.Height()); - - BoundingBox bb; - bb.left = offset_x; - bb.top = offset_y; - bb.width = mask.Width(); - bb.height = mask.Height(); +bool getMaskFromLabels(aliceVision::image::Image & mask, image::Image & labels, IndexT index, int offset_x, int offset_y) +{ - if (!loopyCachedImageExtract(extractedLabels, labels, bb)) + for (int i = 0; i < mask.Height(); i++) { - return false; - } + int y = i + offset_y; - for (int i = 0; i < extractedLabels.Height(); i++) - { - for (int j = 0; j < extractedLabels.Width(); j++) + for (int j = 0; j < mask.Width(); j++) { - if (extractedLabels(i, j) == index) + int x = j + offset_x; + mask(i, j) = 0; + + if (y < 0 || y >= labels.Height()) continue; + if (x < 0 || x >= labels.Width()) continue; + + if (labels(y, x) == index) { mask(i, j) = 1.0f; } - else - { - mask(i, j) = 0.0f; - } } } diff --git a/src/aliceVision/panorama/seams.hpp b/src/aliceVision/panorama/seams.hpp index d78f955b71..7f41c4ea09 100644 --- a/src/aliceVision/panorama/seams.hpp +++ b/src/aliceVision/panorama/seams.hpp @@ -12,33 +12,33 @@ bool drawBorders(CachedImage & inout, const aliceVision::imag bool drawSeams(CachedImage& inout, CachedImage& labels); -bool getMaskFromLabels(aliceVision::image::Image & mask, CachedImage & labels, IndexT index, size_t offset_x, size_t offset_y); +bool getMaskFromLabels(aliceVision::image::Image & mask, image::Image & labels, IndexT index, int offset_x, int offset_y); class WTASeams { public: WTASeams(size_t outputWidth, size_t outputHeight) - : _panoramaWidth(outputWidth) + : _weights(outputWidth, outputHeight, true, 0.0f) + , _labels(outputWidth, outputHeight, true, UndefinedIndexT) + , _panoramaWidth(outputWidth) , _panoramaHeight(outputHeight) { } virtual ~WTASeams() = default; - - bool initialize(image::TileCacheManager::shared_ptr & cacheManager); - + bool append(const aliceVision::image::Image& inputMask, const aliceVision::image::Image& inputWeights, IndexT currentIndex, size_t offset_x, size_t offset_y); - CachedImage & getLabels() + image::Image & getLabels() { return _labels; } private: - CachedImage _weights; - CachedImage _labels; + image::Image _weights; + image::Image _labels; int _panoramaWidth; int _panoramaHeight; diff --git a/src/software/pipeline/CMakeLists.txt b/src/software/pipeline/CMakeLists.txt index 82e9995b6f..151d17f4bb 100644 --- a/src/software/pipeline/CMakeLists.txt +++ b/src/software/pipeline/CMakeLists.txt @@ -147,6 +147,16 @@ if(ALICEVISION_BUILD_SFM) aliceVision_panorama ${Boost_LIBRARIES} ) + alicevision_add_software(aliceVision_panoramaSeams + SOURCE main_panoramaSeams.cpp + FOLDER ${FOLDER_SOFTWARE_PIPELINE} + LINKS aliceVision_system + aliceVision_image + aliceVision_sfmData + aliceVision_sfmDataIO + aliceVision_panorama + ${Boost_LIBRARIES} + ) alicevision_add_software(aliceVision_panoramaInit SOURCE main_panoramaInit.cpp FOLDER ${FOLDER_SOFTWARE_PIPELINE} diff --git a/src/software/pipeline/main_panoramaCompositing.cpp b/src/software/pipeline/main_panoramaCompositing.cpp index 0c94a2e1d2..75403ec25c 100644 --- a/src/software/pipeline/main_panoramaCompositing.cpp +++ b/src/software/pipeline/main_panoramaCompositing.cpp @@ -4,12 +4,10 @@ // v. 2.0. If a copy of the MPL was not distributed with this file, // You can obtain one at https://mozilla.org/MPL/2.0/. -#include +#include #include #include #include -#include -#include // Input and geometry #include @@ -46,135 +44,106 @@ namespace po = boost::program_options; namespace bpt = boost::property_tree; namespace fs = boost::filesystem; -bool buildMap(BoundingBoxPanoramaMap & map, const std::vector> & views, const std::string & inputPath, const std::unique_ptr & compositer) +size_t getCompositingOptimalScale(int width, int height) { - for (const auto& viewIt : views) + /* + Look for the smallest scale such that the image is not smaller than the + convolution window size. + minsize / 2^x = 5 + minsize / 5 = 2^x + x = log2(minsize/5) + */ + + size_t minsize = std::min(width, height); + size_t gaussianFilterRadius = 2; + + int gaussianFilterSize = 1 + 2 * 2; + + size_t optimal_scale = size_t(floor(std::log2(double(minsize) / gaussianFilterSize))); + + return (optimal_scale - 1); +} + +std::unique_ptr buildMap(const sfmData::SfMData & sfmData, const std::string & inputPath, const size_t borderSize) +{ + if (sfmData.getViews().size() == 0) { - // Load mask - const std::string maskPath = (fs::path(inputPath) / (std::to_string(viewIt->getViewId()) + "_mask.exr")).string(); - ALICEVISION_LOG_TRACE("Load mask with path " << maskPath); - image::Image mask; - image::readImage(maskPath, mask, image::EImageColorSpace::NO_CONVERSION); + return nullptr; + } - oiio::ParamValueList metadata = image::readImageMetadata(maskPath); + size_t min_scale = std::numeric_limits::max(); + std::vector> listBoundingBox; + std::pair panoramaSize; + for (const auto& viewIt : sfmData.getViews()) + { + // Load mask + const std::string maskPath = (fs::path(inputPath) / (std::to_string(viewIt.first) + "_mask.exr")).string(); + ALICEVISION_LOG_TRACE("Load metadata of mask with path " << maskPath); + + int width = 0; + int height = 0; + oiio::ParamValueList metadata = image::readImageMetadata(maskPath, width, height); const std::size_t offsetX = metadata.find("AliceVision:offsetX")->get_int(); const std::size_t offsetY = metadata.find("AliceVision:offsetY")->get_int(); + panoramaSize.first = metadata.find("AliceVision:panoramaWidth")->get_int(); + panoramaSize.second = metadata.find("AliceVision:panoramaHeight")->get_int(); BoundingBox bb; bb.left = offsetX; bb.top = offsetY; - bb.width = mask.Width(); - bb.height = mask.Height(); - - int border = compositer->getBorderSize(); - int scale = compositer->getOptimalScale(mask.Width(), mask.Height()); + bb.width = width; + bb.height = height; - if (!map.append(viewIt->getViewId(), bb, border, scale)) + listBoundingBox.push_back(std::make_pair(viewIt.first, bb)); + size_t scale = getCompositingOptimalScale(width, height); + if (scale < min_scale) { - return false; + min_scale = scale; } } - return true; + + std::unique_ptr ret(new PanoramaMap(panoramaSize.first, panoramaSize.second, min_scale, borderSize)); + for (const auto & bbitem : listBoundingBox) + { + ret->append(bbitem.first, bbitem.second); + } + + return ret; } -bool computeWTALabels(CachedImage & labels, image::TileCacheManager::shared_ptr& cacheManager, const std::vector> & views, const std::string & inputPath, std::pair & panoramaSize) +bool computeWTALabels(image::Image & labels, std::list & views, const std::string & inputPath, const BoundingBox & boundingBox) { ALICEVISION_LOG_INFO("Estimating initial labels for panorama"); - WTASeams seams(panoramaSize.first, panoramaSize.second); - - if (!seams.initialize(cacheManager)) - { - return false; - } + WTASeams seams(boundingBox.width, boundingBox.height); - for (const auto& viewIt : views) + for (const IndexT & currentId : views) { - IndexT viewId = viewIt->getViewId(); - // Load mask - const std::string maskPath = (fs::path(inputPath) / (std::to_string(viewId) + "_mask.exr")).string(); + const std::string maskPath = (fs::path(inputPath) / (std::to_string(currentId) + "_mask.exr")).string(); ALICEVISION_LOG_TRACE("Load mask with path " << maskPath); image::Image mask; image::readImage(maskPath, mask, image::EImageColorSpace::NO_CONVERSION); // Get offset oiio::ParamValueList metadata = image::readImageMetadata(maskPath); - const std::size_t offsetX = metadata.find("AliceVision:offsetX")->get_int(); - const std::size_t offsetY = metadata.find("AliceVision:offsetY")->get_int(); + const int offsetX = metadata.find("AliceVision:offsetX")->get_int(); + const int offsetY = metadata.find("AliceVision:offsetY")->get_int(); // Load Weights - const std::string weightsPath = (fs::path(inputPath) / (std::to_string(viewId) + "_weight.exr")).string(); + const std::string weightsPath = (fs::path(inputPath) / (std::to_string(currentId) + "_weight.exr")).string(); ALICEVISION_LOG_TRACE("Load weights with path " << weightsPath); image::Image weights; image::readImage(weightsPath, weights, image::EImageColorSpace::NO_CONVERSION); - - if (!seams.append(mask, weights, viewId, offsetX, offsetY)) - { - return false; - } - } - - labels = seams.getLabels(); - - return true; -} - -bool computeGCLabels(CachedImage & labels, image::TileCacheManager::shared_ptr& cacheManager, const std::vector> & views, const std::string & inputPath, std::pair & panoramaSize, int smallestViewScale) -{ - ALICEVISION_LOG_INFO("Estimating smart seams for panorama"); - - int pyramidSize = 1 + std::max(0, smallestViewScale - 1); - ALICEVISION_LOG_INFO("Graphcut pyramid size is " << pyramidSize); - - HierarchicalGraphcutSeams seams(cacheManager, panoramaSize.first, panoramaSize.second, pyramidSize); - - if (!seams.initialize()) - { - return false; - } - - if (!seams.setOriginalLabels(labels)) - { - return false; - } - - for (const auto& viewIt : views) - { - IndexT viewId = viewIt->getViewId(); - - // Load mask - const std::string maskPath = (fs::path(inputPath) / (std::to_string(viewId) + "_mask.exr")).string(); - ALICEVISION_LOG_TRACE("Load mask with path " << maskPath); - image::Image mask; - image::readImage(maskPath, mask, image::EImageColorSpace::NO_CONVERSION); - - // Load Color - const std::string colorsPath = (fs::path(inputPath) / (std::to_string(viewId) + ".exr")).string(); - ALICEVISION_LOG_TRACE("Load colors with path " << colorsPath); - image::Image colors; - image::readImage(colorsPath, colors, image::EImageColorSpace::NO_CONVERSION); - - // Get offset - oiio::ParamValueList metadata = image::readImageMetadata(maskPath); - const std::size_t offsetX = metadata.find("AliceVision:offsetX")->get_int(); - const std::size_t offsetY = metadata.find("AliceVision:offsetY")->get_int(); - - - // Append to graph cut - if (!seams.append(colors, mask, viewId, offsetX, offsetY)) + + if (!seams.append(mask, weights, currentId, offsetX - boundingBox.left, offsetY - boundingBox.top)) { return false; } } - if (!seams.process()) - { - return false; - } - labels = seams.getLabels(); return true; @@ -184,16 +153,9 @@ int aliceVision_main(int argc, char** argv) { std::string sfmDataFilepath; std::string warpingFolder; - std::string outputPanorama; + std::string outputFolder; std::string temporaryCachePath; - std::string compositerType = "multiband"; - std::string overlayType = "none"; - bool useGraphCut = true; - bool useSeams = false; - bool useWeights = false; - bool showBorders = false; - bool showSeams = false; image::EStorageDataType storageDataType = image::EStorageDataType::Float; @@ -209,26 +171,21 @@ int aliceVision_main(int argc, char** argv) requiredParams.add_options() ("input,i", po::value(&sfmDataFilepath)->required(), "Input sfmData.") ("warpingFolder,w", po::value(&warpingFolder)->required(), "Folder with warped images.") - ("output,o", po::value(&outputPanorama)->required(), "Path of the output panorama.") + ("output,o", po::value(&outputFolder)->required(), "Path of the output panorama.") ("cacheFolder,f", po::value(&temporaryCachePath)->required(), "Path of the temporary cache."); allParams.add(requiredParams); // Description of optional parameters po::options_description optionalParams("Optional parameters"); - optionalParams.add_options()("compositerType,c", po::value(&compositerType)->required(), - "Compositer Type [replace, alpha, multiband].")( - "overlayType,c", po::value(&overlayType)->required(), "Overlay Type [none, borders, seams, all].")( - "useGraphCut,g", po::value(&useGraphCut)->default_value(useGraphCut), - "Do we use graphcut for ghost removal ?")( - "storageDataType", po::value(&storageDataType)->default_value(storageDataType), - ("Storage data type: " + image::EStorageDataType_informations()).c_str()); + optionalParams.add_options() + ("compositerType,c", po::value(&compositerType)->required(), "Compositer Type [replace, alpha, multiband].") + ("storageDataType", po::value(&storageDataType)->default_value(storageDataType), ("Storage data type: " + image::EStorageDataType_informations()).c_str()); allParams.add(optionalParams); // Setup log level given command line po::options_description logParams("Log parameters"); - logParams.add_options()("verboseLevel,v", - po::value(&verboseLevel)->default_value(verboseLevel), - "verbosity level (fatal, error, warning, info, debug, trace)."); + logParams.add_options() + ("verboseLevel,v", po::value(&verboseLevel)->default_value(verboseLevel), "verbosity level (fatal, error, warning, info, debug, trace)."); allParams.add(logParams); // Effectively parse command line given parse options @@ -265,259 +222,104 @@ int aliceVision_main(int argc, char** argv) // load input scene sfmData::SfMData sfmData; - if(!sfmDataIO::Load(sfmData, sfmDataFilepath, - sfmDataIO::ESfMData(sfmDataIO::VIEWS | sfmDataIO::EXTRINSICS | sfmDataIO::INTRINSICS))) + if(!sfmDataIO::Load(sfmData, sfmDataFilepath, sfmDataIO::ESfMData(sfmDataIO::VIEWS | sfmDataIO::EXTRINSICS | sfmDataIO::INTRINSICS))) { ALICEVISION_LOG_ERROR("The input file '" + sfmDataFilepath + "' cannot be read"); return EXIT_FAILURE; } - int tileSize; - std::pair panoramaSize; + const size_t borderSize = 2; + std::unique_ptr panoramaMap = buildMap(sfmData, warpingFolder, borderSize); + if (sfmData.getViews().size() == 0) { - const IndexT viewId = *sfmData.getValidViews().begin(); - const std::string viewFilepath = (fs::path(warpingFolder) / (std::to_string(viewId) + ".exr")).string(); - ALICEVISION_LOG_TRACE("Read panorama size from file: " << viewFilepath); - - oiio::ParamValueList metadata = image::readImageMetadata(viewFilepath); - panoramaSize.first = metadata.find("AliceVision:panoramaWidth")->get_int(); - panoramaSize.second = metadata.find("AliceVision:panoramaHeight")->get_int(); - tileSize = metadata.find("AliceVision:tileSize")->get_int(); - - if(panoramaSize.first == 0 || panoramaSize.second == 0) - { - ALICEVISION_LOG_ERROR("The output panorama size is empty."); - return EXIT_FAILURE; - } - - if(tileSize == 0) - { - ALICEVISION_LOG_ERROR("no information on tileSize"); - return EXIT_FAILURE; - } - - ALICEVISION_LOG_INFO("Output panorama size set to " << panoramaSize.first << "x" << panoramaSize.second); - } - - if(!temporaryCachePath.empty() && !fs::exists(temporaryCachePath)) - { - fs::create_directory(temporaryCachePath); + ALICEVISION_LOG_ERROR("No valid views"); + return EXIT_FAILURE; } - // Create a cache manager - image::TileCacheManager::shared_ptr cacheManager = image::TileCacheManager::create(temporaryCachePath, 256, 256, 65536); - if(!cacheManager) + IndexT viewReference = sfmData.getViews().begin()->first; + std::list overlaps; + if (!panoramaMap->getOverlaps(overlaps, viewReference)) { - ALICEVISION_LOG_ERROR("Error creating the cache manager"); + ALICEVISION_LOG_ERROR("Problem analyzing neighboorhood"); return EXIT_FAILURE; } + overlaps.push_back(viewReference); - // Configure the cache manager memory - system::MemoryInfo memInfo = system::getMemoryInfo(); - const double convertionGb = std::pow(2,30); - ALICEVISION_LOG_INFO("Available RAM is " << std::setw(5) << memInfo.availableRam / convertionGb << "GB (" << memInfo.availableRam << " octets)."); - cacheManager->setMaxMemory(memInfo.freeRam); - - if (overlayType == "borders" || overlayType == "all") + BoundingBox referenceBoundingBox; + if (!panoramaMap->getEnhancedBoundingBox(referenceBoundingBox, viewReference)) { - showBorders = true; - } - - if (overlayType == "seams" || overlayType == "all") { - showSeams = true; + ALICEVISION_LOG_ERROR("Problem getting reference bounding box"); + return EXIT_FAILURE; } std::unique_ptr compositer; - if (compositerType == "multiband") - { - compositer = std::unique_ptr(new LaplacianCompositer(cacheManager, panoramaSize.first, panoramaSize.second)); - useSeams = true; - useWeights = false; - } - else if (compositerType == "alpha") - { - compositer = std::unique_ptr(new AlphaCompositer(cacheManager, panoramaSize.first, panoramaSize.second)); - useGraphCut = false; - useSeams = false; - useWeights = true; - } - else - { - compositer = std::unique_ptr(new Compositer(cacheManager, panoramaSize.first, panoramaSize.second)); - useGraphCut = false; - useSeams = false; - useWeights = false; - } - + compositer = std::unique_ptr(new LaplacianCompositer(referenceBoundingBox.width, referenceBoundingBox.height)); - if (!compositer->initialize()) + if (!compositer->initialize()) { - ALICEVISION_LOG_ERROR("Failed to initialize compositer"); - return EXIT_FAILURE; + ALICEVISION_LOG_ERROR("Error initializing panorama"); + return false; } - //Get a list of views ordered by their image scale - size_t smallestScale = 0; - std::vector> viewOrderedByScale; + image::Image labels; + if (!computeWTALabels(labels, overlaps, warpingFolder, referenceBoundingBox)) { - std::map>> mapViewsScale; - for(const auto & it : sfmData.getViews()) - { - auto view = it.second; - IndexT viewId = view->getViewId(); - - if(!sfmData.isPoseAndIntrinsicDefined(view.get())) - { - // skip unreconstructed views - continue; - } - - // Load mask - const std::string maskPath = (fs::path(warpingFolder) / (std::to_string(viewId) + "_mask.exr")).string(); - image::Image mask; - image::readImage(maskPath, mask, image::EImageColorSpace::NO_CONVERSION); - - //Estimate scale - size_t scale = compositer->getOptimalScale(mask.Width(), mask.Height()); - mapViewsScale[scale].push_back(it.second); - } - - if (mapViewsScale.size() == 0) - { - ALICEVISION_LOG_ERROR("No valid view"); - return EXIT_FAILURE; - } - - smallestScale = mapViewsScale.begin()->first; - - for (auto scaledList : mapViewsScale) - { - for (auto item : scaledList.second) - { - viewOrderedByScale.push_back(item); - } - } + ALICEVISION_LOG_ERROR("Error estimating panorama labels for this input"); + return false; } - ALICEVISION_LOG_INFO(viewOrderedByScale.size() << " views to process"); - - CachedImage labels; - if (useSeams) - { - if (!computeWTALabels(labels, cacheManager, viewOrderedByScale, warpingFolder, panoramaSize)) - { - ALICEVISION_LOG_ERROR("Error computing initial labels"); - return EXIT_FAILURE; - } - - if (useGraphCut) - { - if (!computeGCLabels(labels, cacheManager, viewOrderedByScale, warpingFolder, panoramaSize, smallestScale)) - { - ALICEVISION_LOG_ERROR("Error computing graph cut labels"); - return EXIT_FAILURE; - } - } - } + image::writeImage("/home/servantf/labels.exr", labels, image::EImageColorSpace::LINEAR); - size_t pos = 0; - for(const auto & view : viewOrderedByScale) + int pos = 0; + for (IndexT viewCurrent : overlaps) { - IndexT viewId = view->getViewId(); - pos++; - - ALICEVISION_LOG_INFO("Processing input " << pos << "/" << viewOrderedByScale.size()); + ALICEVISION_LOG_INFO("Processing input " << pos << "/" << overlaps.size()); // Load image and convert it to linear colorspace - const std::string imagePath = (fs::path(warpingFolder) / (std::to_string(viewId) + ".exr")).string(); + const std::string imagePath = (fs::path(warpingFolder) / (std::to_string(viewCurrent) + ".exr")).string(); ALICEVISION_LOG_TRACE("Load image with path " << imagePath); image::Image source; image::readImage(imagePath, source, image::EImageColorSpace::NO_CONVERSION); - // Retrieve position of image in panorama - oiio::ParamValueList metadata = image::readImageMetadata(imagePath); - const std::size_t offsetX = metadata.find("AliceVision:offsetX")->get_int(); - const std::size_t offsetY = metadata.find("AliceVision:offsetY")->get_int(); - // Load mask - const std::string maskPath = (fs::path(warpingFolder) / (std::to_string(viewId) + "_mask.exr")).string(); + const std::string maskPath = (fs::path(warpingFolder) / (std::to_string(viewCurrent) + "_mask.exr")).string(); ALICEVISION_LOG_TRACE("Load mask with path " << maskPath); image::Image mask; image::readImage(maskPath, mask, image::EImageColorSpace::NO_CONVERSION); - // Load Weights - image::Image weights; - if (useWeights) - { - const std::string weightsPath = (fs::path(warpingFolder) / (std::to_string(viewId) + "_weight.exr")).string(); - ALICEVISION_LOG_TRACE("Load weights with path " << weightsPath); - image::readImage(weightsPath, weights, image::EImageColorSpace::NO_CONVERSION); - } - else if (useSeams) - { - weights = image::Image(mask.Width(), mask.Height()); - if (!getMaskFromLabels(weights, labels, viewId, offsetX, offsetY)) - { - ALICEVISION_LOG_ERROR("Error estimating seams image"); - return EXIT_FAILURE; - } - } + // Retrieve position of image in panorama + oiio::ParamValueList metadata = image::readImageMetadata(imagePath); + const int offsetX = metadata.find("AliceVision:offsetX")->get_int(); + const int offsetY = metadata.find("AliceVision:offsetY")->get_int(); - if (!compositer->append(source, mask, weights, offsetX, offsetY)) + image::Image weights; + /*const std::string weightsPath = (fs::path(warpingFolder) / (std::to_string(viewCurrent) + "_weight.exr")).string(); + ALICEVISION_LOG_TRACE("Load weights with path " << weightsPath); + image::readImage(weightsPath, weights, image::EImageColorSpace::NO_CONVERSION);*/ + weights = image::Image(mask.Width(), mask.Height()); + if (!getMaskFromLabels(weights, labels, viewCurrent, offsetX - referenceBoundingBox.left, offsetY - referenceBoundingBox.top)) { + ALICEVISION_LOG_ERROR("Error estimating seams image"); return EXIT_FAILURE; } - } - - ALICEVISION_LOG_INFO("Final processing"); - if (!compositer->terminate()) - { - ALICEVISION_LOG_ERROR("Error terminating panorama"); - return EXIT_FAILURE; - } - - if (showBorders) - { - ALICEVISION_LOG_INFO("Drawing borders on panorama"); - for(const auto & view : viewOrderedByScale) + if (!compositer->append(source, mask, weights, offsetX - referenceBoundingBox.left, offsetY - referenceBoundingBox.top)) { - IndexT viewId = view->getViewId(); - - // Load mask - const std::string maskPath = (fs::path(warpingFolder) / (std::to_string(viewId) + "_mask.exr")).string(); - ALICEVISION_LOG_TRACE("Load mask with path " << maskPath); - image::Image mask; - image::readImage(maskPath, mask, image::EImageColorSpace::NO_CONVERSION); - - oiio::ParamValueList metadata = image::readImageMetadata(maskPath); - const std::size_t offsetX = metadata.find("AliceVision:offsetX")->get_int(); - const std::size_t offsetY = metadata.find("AliceVision:offsetY")->get_int(); - - if (!compositer->drawBorders(mask, offsetX, offsetY)) - { - ALICEVISION_LOG_ERROR("Failed to draw borders"); - return EXIT_FAILURE; - } - } + ALICEVISION_LOG_INFO("Error in compositer append"); + return EXIT_FAILURE; + } + + pos++; } - if (showSeams && useSeams) + if (!compositer->terminate()) { - ALICEVISION_LOG_INFO("Drawing seams on panorama"); - - if (!compositer->drawSeams(labels)) - { - ALICEVISION_LOG_ERROR("Failed to draw borders"); - return EXIT_FAILURE; - } + ALICEVISION_LOG_ERROR("Error terminating panorama"); } - ALICEVISION_LOG_INFO("Saving panorama to file"); - if (!compositer->save(outputPanorama, storageDataType)) + if (!compositer->save("/home/servantf/test.exr", image::EStorageDataType::HalfFinite)) { - ALICEVISION_LOG_ERROR("Impossible to save file"); - return EXIT_FAILURE; + ALICEVISION_LOG_ERROR("Error terminating panorama"); } return EXIT_SUCCESS; diff --git a/src/software/pipeline/main_panoramaSeams.cpp b/src/software/pipeline/main_panoramaSeams.cpp new file mode 100644 index 0000000000..431ed05355 --- /dev/null +++ b/src/software/pipeline/main_panoramaSeams.cpp @@ -0,0 +1,356 @@ +// This file is part of the AliceVision project. +// Copyright (c) 2020 AliceVision contributors. +// This Source Code Form is subject to the terms of the Mozilla Public License, +// v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at https://mozilla.org/MPL/2.0/. + + +#include + +// Input and geometry +#include +#include + +// Image +#include +#include + +// System +#include +#include + +// Reading command line options +#include +#include +#include + +// IO +#include +#include +#include +#include +#include + +// These constants define the current software version. +// They must be updated when the command line is changed. +#define ALICEVISION_SOFTWARE_VERSION_MAJOR 1 +#define ALICEVISION_SOFTWARE_VERSION_MINOR 0 + +using namespace aliceVision; + +namespace po = boost::program_options; +namespace bpt = boost::property_tree; +namespace fs = boost::filesystem; + +bool computeWTALabels(image::Image & labels, const std::vector> & views, const std::string & inputPath, std::pair & panoramaSize) +{ + ALICEVISION_LOG_INFO("Estimating initial labels for panorama"); + + WTASeams seams(panoramaSize.first, panoramaSize.second); + + for (const auto& viewIt : views) + { + IndexT viewId = viewIt->getViewId(); + + // Load mask + const std::string maskPath = (fs::path(inputPath) / (std::to_string(viewId) + "_mask.exr")).string(); + ALICEVISION_LOG_TRACE("Load mask with path " << maskPath); + image::Image mask; + image::readImage(maskPath, mask, image::EImageColorSpace::NO_CONVERSION); + + // Get offset + oiio::ParamValueList metadata = image::readImageMetadata(maskPath); + const std::size_t offsetX = metadata.find("AliceVision:offsetX")->get_int(); + const std::size_t offsetY = metadata.find("AliceVision:offsetY")->get_int(); + + // Load Weights + const std::string weightsPath = (fs::path(inputPath) / (std::to_string(viewId) + "_weight.exr")).string(); + ALICEVISION_LOG_TRACE("Load weights with path " << weightsPath); + image::Image weights; + image::readImage(weightsPath, weights, image::EImageColorSpace::NO_CONVERSION); + + if (!seams.append(mask, weights, viewId, offsetX, offsetY)) + { + return false; + } + } + + labels = seams.getLabels(); + + return true; +} + +bool computeGCLabels(CachedImage & labels, image::TileCacheManager::shared_ptr& cacheManager, const std::vector> & views, const std::string & inputPath, std::pair & panoramaSize, int smallestViewScale) +{ + ALICEVISION_LOG_INFO("Estimating smart seams for panorama"); + + int pyramidSize = 1 + std::max(0, smallestViewScale - 1); + ALICEVISION_LOG_INFO("Graphcut pyramid size is " << pyramidSize); + + HierarchicalGraphcutSeams seams(cacheManager, panoramaSize.first, panoramaSize.second, pyramidSize); + + + if (!seams.initialize()) + { + return false; + } + + if (!seams.setOriginalLabels(labels)) + { + return false; + } + + for (const auto& viewIt : views) + { + IndexT viewId = viewIt->getViewId(); + + // Load mask + const std::string maskPath = (fs::path(inputPath) / (std::to_string(viewId) + "_mask.exr")).string(); + ALICEVISION_LOG_TRACE("Load mask with path " << maskPath); + image::Image mask; + image::readImage(maskPath, mask, image::EImageColorSpace::NO_CONVERSION); + + // Load Color + const std::string colorsPath = (fs::path(inputPath) / (std::to_string(viewId) + ".exr")).string(); + ALICEVISION_LOG_TRACE("Load colors with path " << colorsPath); + image::Image colors; + image::readImage(colorsPath, colors, image::EImageColorSpace::NO_CONVERSION); + + // Get offset + oiio::ParamValueList metadata = image::readImageMetadata(maskPath); + const std::size_t offsetX = metadata.find("AliceVision:offsetX")->get_int(); + const std::size_t offsetY = metadata.find("AliceVision:offsetY")->get_int(); + + + // Append to graph cut + if (!seams.append(colors, mask, viewId, offsetX, offsetY)) + { + return false; + } + } + + if (!seams.process()) + { + return false; + } + + labels = seams.getLabels(); + + return true; +} + +size_t getGraphcutOptimalScale(int width, int height) +{ + /* + Look for the smallest scale such that the image is not smaller than the + convolution window size. + minsize / 2^x = 5 + minsize / 5 = 2^x + x = log2(minsize/5) + */ + + size_t minsize = std::min(width, height); + size_t gaussianFilterRadius = 2; + + int gaussianFilterSize = 1 + 2 * gaussianFilterRadius; + + size_t optimal_scale = size_t(floor(std::log2(double(minsize) / gaussianFilterSize))); + + return (optimal_scale - 1/*Security*/); +} + +int aliceVision_main(int argc, char** argv) +{ + std::string sfmDataFilepath; + std::string warpingFolder; + std::string outputLabels; + std::string temporaryCachePath; + + bool useGraphCut = true; + image::EStorageDataType storageDataType = image::EStorageDataType::Float; + + system::EVerboseLevel verboseLevel = system::Logger::getDefaultVerboseLevel(); + + // Program description + po::options_description allParams( + "Perform panorama stiching of cameras around a nodal point for 360° panorama creation. \n" + "AliceVision PanoramaCompositing"); + + // Description of mandatory parameters + po::options_description requiredParams("Required parameters"); + requiredParams.add_options() + ("input,i", po::value(&sfmDataFilepath)->required(), "Input sfmData.") + ("warpingFolder,w", po::value(&warpingFolder)->required(), "Folder with warped images.") + ("output,o", po::value(&outputLabels)->required(), "Path of the output labels.") + ("cacheFolder,f", po::value(&temporaryCachePath)->required(), "Path of the temporary cache."); + allParams.add(requiredParams); + + // Description of optional parameters + po::options_description optionalParams("Optional parameters"); + optionalParams.add_options() + ("useGraphCut,g", po::value(&useGraphCut)->default_value(useGraphCut), "Do we use graphcut for ghost removal ?"); + allParams.add(optionalParams); + + // Setup log level given command line + po::options_description logParams("Log parameters"); + logParams.add_options()("verboseLevel,v", + po::value(&verboseLevel)->default_value(verboseLevel), + "verbosity level (fatal, error, warning, info, debug, trace)."); + allParams.add(logParams); + + // Effectively parse command line given parse options + po::variables_map vm; + try + { + po::store(po::parse_command_line(argc, argv, allParams), vm); + + if(vm.count("help") || (argc == 1)) + { + ALICEVISION_COUT(allParams); + return EXIT_SUCCESS; + } + po::notify(vm); + } + catch(boost::program_options::required_option& e) + { + ALICEVISION_CERR("ERROR: " << e.what()); + ALICEVISION_COUT("Usage:\n\n" << allParams); + return EXIT_FAILURE; + } + catch(boost::program_options::error& e) + { + ALICEVISION_CERR("ERROR: " << e.what()); + ALICEVISION_COUT("Usage:\n\n" << allParams); + return EXIT_FAILURE; + } + + ALICEVISION_COUT("Program called with the following parameters:"); + ALICEVISION_COUT(vm); + + // Set verbose level given command line + system::Logger::get()->setLogLevel(verboseLevel); + + // load input scene + sfmData::SfMData sfmData; + if(!sfmDataIO::Load(sfmData, sfmDataFilepath, + sfmDataIO::ESfMData(sfmDataIO::VIEWS | sfmDataIO::EXTRINSICS | sfmDataIO::INTRINSICS))) + { + ALICEVISION_LOG_ERROR("The input file '" + sfmDataFilepath + "' cannot be read"); + return EXIT_FAILURE; + } + + int tileSize; + std::pair panoramaSize; + { + const IndexT viewId = *sfmData.getValidViews().begin(); + const std::string viewFilepath = (fs::path(warpingFolder) / (std::to_string(viewId) + ".exr")).string(); + ALICEVISION_LOG_TRACE("Read panorama size from file: " << viewFilepath); + + oiio::ParamValueList metadata = image::readImageMetadata(viewFilepath); + panoramaSize.first = metadata.find("AliceVision:panoramaWidth")->get_int(); + panoramaSize.second = metadata.find("AliceVision:panoramaHeight")->get_int(); + tileSize = metadata.find("AliceVision:tileSize")->get_int(); + + if(panoramaSize.first == 0 || panoramaSize.second == 0) + { + ALICEVISION_LOG_ERROR("The output panorama size is empty."); + return EXIT_FAILURE; + } + + if(tileSize == 0) + { + ALICEVISION_LOG_ERROR("no information on tileSize"); + return EXIT_FAILURE; + } + + ALICEVISION_LOG_INFO("Output labels size set to " << panoramaSize.first << "x" << panoramaSize.second); + } + + if(!temporaryCachePath.empty() && !fs::exists(temporaryCachePath)) + { + fs::create_directory(temporaryCachePath); + } + + // Create a cache manager + image::TileCacheManager::shared_ptr cacheManager = image::TileCacheManager::create(temporaryCachePath, 256, 256, 65536); + if(!cacheManager) + { + ALICEVISION_LOG_ERROR("Error creating the cache manager"); + return EXIT_FAILURE; + } + + // Configure the cache manager memory + system::MemoryInfo memInfo = system::getMemoryInfo(); + const double convertionGb = std::pow(2,30); + ALICEVISION_LOG_INFO("Available RAM is " << std::setw(5) << memInfo.availableRam / convertionGb << "GB (" << memInfo.availableRam << " octets)."); + cacheManager->setMaxMemory(1024ll * 1024ll * 1024ll * 6ll); + + + //Get a list of views ordered by their image scale + size_t smallestScale = 0; + std::vector> viewOrderedByScale; + { + std::map>> mapViewsScale; + for(const auto & it : sfmData.getViews()) + { + auto view = it.second; + IndexT viewId = view->getViewId(); + + if(!sfmData.isPoseAndIntrinsicDefined(view.get())) + { + // skip unreconstructed views + continue; + } + + // Load mask + const std::string maskPath = (fs::path(warpingFolder) / (std::to_string(viewId) + "_mask.exr")).string(); + image::Image mask; + image::readImage(maskPath, mask, image::EImageColorSpace::NO_CONVERSION); + + //Estimate scale + size_t scale = getGraphcutOptimalScale(mask.Width(), mask.Height()); + mapViewsScale[scale].push_back(it.second); + } + + if (mapViewsScale.size() == 0) + { + ALICEVISION_LOG_ERROR("No valid view"); + return EXIT_FAILURE; + } + + smallestScale = mapViewsScale.begin()->first; + for (auto scaledList : mapViewsScale) + { + for (auto item : scaledList.second) + { + viewOrderedByScale.push_back(item); + } + } + } + + ALICEVISION_LOG_INFO(viewOrderedByScale.size() << " views to process"); + + image::Image labels; + if (!computeWTALabels(labels, viewOrderedByScale, warpingFolder, panoramaSize)) + { + ALICEVISION_LOG_ERROR("Error computing initial labels"); + return EXIT_FAILURE; + } + + /*if (useGraphCut) + { + if (!computeGCLabels(labels, cacheManager, viewOrderedByScale, warpingFolder, panoramaSize, smallestScale)) + { + ALICEVISION_LOG_ERROR("Error computing graph cut labels"); + return EXIT_FAILURE; + } + } + + if (!labels.writeImage(outputLabels)) + { + ALICEVISION_LOG_ERROR("Error writing labels to disk"); + return EXIT_FAILURE; + }*/ + + return EXIT_SUCCESS; +} From f7d08c12d019d826c1746fc8915ff7fe08c85f03 Mon Sep 17 00:00:00 2001 From: Fabien Date: Mon, 14 Dec 2020 15:10:47 +0100 Subject: [PATCH 46/79] [panorama] parallelize compositing --- src/aliceVision/panorama/compositer.hpp | 5 - .../panorama/laplacianCompositer.hpp | 47 +---- src/aliceVision/panorama/panoramaMap.hpp | 5 + .../pipeline/main_panoramaCompositing.cpp | 178 +++++++++++------- 4 files changed, 114 insertions(+), 121 deletions(-) diff --git a/src/aliceVision/panorama/compositer.hpp b/src/aliceVision/panorama/compositer.hpp index dd377fb058..f22d0184fe 100644 --- a/src/aliceVision/panorama/compositer.hpp +++ b/src/aliceVision/panorama/compositer.hpp @@ -88,11 +88,6 @@ class Compositer return true; } - virtual size_t getOptimalScale(int width, int height) const - { - return 0; - } - virtual int getBorderSize() const { return 0; diff --git a/src/aliceVision/panorama/laplacianCompositer.hpp b/src/aliceVision/panorama/laplacianCompositer.hpp index 365248bb9c..98ff639ffa 100644 --- a/src/aliceVision/panorama/laplacianCompositer.hpp +++ b/src/aliceVision/panorama/laplacianCompositer.hpp @@ -10,10 +10,10 @@ namespace aliceVision class LaplacianCompositer : public Compositer { public: - LaplacianCompositer(size_t outputWidth, size_t outputHeight) + LaplacianCompositer(size_t outputWidth, size_t outputHeight, size_t scale) : Compositer(outputWidth, outputHeight) - , _pyramidPanorama(outputWidth, outputHeight, 1) - , _bands(1) + , _pyramidPanorama(outputWidth, outputHeight, scale + 1) + , _bands(scale + 1) { } @@ -27,25 +27,6 @@ class LaplacianCompositer : public Compositer return _pyramidPanorama.initialize(); } - virtual size_t getOptimalScale(int width, int height) const - { - /* - Look for the smallest scale such that the image is not smaller than the - convolution window size. - minsize / 2^x = 5 - minsize / 5 = 2^x - x = log2(minsize/5) - */ - - size_t minsize = std::min(width, height); - - int gaussianFilterSize = 1 + 2 * _gaussianFilterRadius; - - size_t optimal_scale = size_t(floor(std::log2(double(minsize) / gaussianFilterSize))); - - return 5;//(optimal_scale - 1/*Security*/); - } - virtual int getBorderSize() const { return _gaussianFilterRadius; @@ -54,28 +35,8 @@ class LaplacianCompositer : public Compositer virtual bool append(const aliceVision::image::Image& color, const aliceVision::image::Image& inputMask, const aliceVision::image::Image& inputWeights, - int offsetX, int offsetY, size_t optimalScale) + int offsetX, int offsetY) { - size_t optimalScale = getOptimalScale(color.Width(), color.Height()); - size_t optimalLevelsCount = optimalScale + 1; - - if(optimalLevelsCount < _bands) - { - ALICEVISION_LOG_ERROR("Decreasing level count !"); - return false; - } - - //If the input scale is more important than previously processed, - // The pyramid must be deepened accordingly - if(optimalLevelsCount > _bands && _bands == 1) - { - _bands = optimalLevelsCount; - if (!_pyramidPanorama.augment(_bands)) - { - return false; - } - } - // Make sure input is compatible with pyramid processing // See comments inside function for details int newOffsetX, newOffsetY; diff --git a/src/aliceVision/panorama/panoramaMap.hpp b/src/aliceVision/panorama/panoramaMap.hpp index a1c613c566..4065d48d04 100644 --- a/src/aliceVision/panorama/panoramaMap.hpp +++ b/src/aliceVision/panorama/panoramaMap.hpp @@ -36,6 +36,11 @@ class PanoramaMap return _panoramaHeight; } + size_t getScale() const + { + return _scale; + } + bool getEnhancedBoundingBox(BoundingBox & bb, const IndexT & id) { if (_map.find(id) == _map.end()) diff --git a/src/software/pipeline/main_panoramaCompositing.cpp b/src/software/pipeline/main_panoramaCompositing.cpp index 75403ec25c..84877c0139 100644 --- a/src/software/pipeline/main_panoramaCompositing.cpp +++ b/src/software/pipeline/main_panoramaCompositing.cpp @@ -156,6 +156,8 @@ int aliceVision_main(int argc, char** argv) std::string outputFolder; std::string temporaryCachePath; std::string compositerType = "multiband"; + int rangeStart = -1; + int rangeSize = 1; image::EStorageDataType storageDataType = image::EStorageDataType::Float; @@ -179,7 +181,9 @@ int aliceVision_main(int argc, char** argv) po::options_description optionalParams("Optional parameters"); optionalParams.add_options() ("compositerType,c", po::value(&compositerType)->required(), "Compositer Type [replace, alpha, multiband].") - ("storageDataType", po::value(&storageDataType)->default_value(storageDataType), ("Storage data type: " + image::EStorageDataType_informations()).c_str()); + ("storageDataType", po::value(&storageDataType)->default_value(storageDataType), ("Storage data type: " + image::EStorageDataType_informations()).c_str()) + ("rangeStart", po::value(&rangeStart)->default_value(rangeStart), "Range image index start.") + ("rangeSize", po::value(&rangeSize)->default_value(rangeSize), "Range size."); allParams.add(optionalParams); // Setup log level given command line @@ -228,98 +232,126 @@ int aliceVision_main(int argc, char** argv) return EXIT_FAILURE; } - const size_t borderSize = 2; - std::unique_ptr panoramaMap = buildMap(sfmData, warpingFolder, borderSize); - if (sfmData.getViews().size() == 0) - { - ALICEVISION_LOG_ERROR("No valid views"); - return EXIT_FAILURE; - } - - IndexT viewReference = sfmData.getViews().begin()->first; - std::list overlaps; - if (!panoramaMap->getOverlaps(overlaps, viewReference)) + // Define range to compute + size_t viewsCount = sfmData.getViews().size(); + if(rangeStart != -1) { - ALICEVISION_LOG_ERROR("Problem analyzing neighboorhood"); - return EXIT_FAILURE; - } - overlaps.push_back(viewReference); + if(rangeStart < 0 || rangeSize < 0 || + std::size_t(rangeStart) > viewsCount) + { + ALICEVISION_LOG_ERROR("Range is incorrect"); + return EXIT_FAILURE; + } - BoundingBox referenceBoundingBox; - if (!panoramaMap->getEnhancedBoundingBox(referenceBoundingBox, viewReference)) - { - ALICEVISION_LOG_ERROR("Problem getting reference bounding box"); - return EXIT_FAILURE; + if(std::size_t(rangeStart + rangeSize) > viewsCount) + { + rangeSize = int(viewsCount) - rangeStart; + } } - - std::unique_ptr compositer; - compositer = std::unique_ptr(new LaplacianCompositer(referenceBoundingBox.width, referenceBoundingBox.height)); - - if (!compositer->initialize()) + else { - ALICEVISION_LOG_ERROR("Error initializing panorama"); - return false; + rangeStart = 0; + rangeSize = int(viewsCount); } - image::Image labels; - if (!computeWTALabels(labels, overlaps, warpingFolder, referenceBoundingBox)) + const size_t borderSize = 2; + std::unique_ptr panoramaMap = buildMap(sfmData, warpingFolder, borderSize); + if (viewsCount == 0) { - ALICEVISION_LOG_ERROR("Error estimating panorama labels for this input"); - return false; + ALICEVISION_LOG_ERROR("No valid views"); + return EXIT_FAILURE; } - image::writeImage("/home/servantf/labels.exr", labels, image::EImageColorSpace::LINEAR); - - int pos = 0; - for (IndexT viewCurrent : overlaps) + int posReference = 0; + for (auto & vIterator : sfmData.getViews()) { - ALICEVISION_LOG_INFO("Processing input " << pos << "/" << overlaps.size()); - - // Load image and convert it to linear colorspace - const std::string imagePath = (fs::path(warpingFolder) / (std::to_string(viewCurrent) + ".exr")).string(); - ALICEVISION_LOG_TRACE("Load image with path " << imagePath); - image::Image source; - image::readImage(imagePath, source, image::EImageColorSpace::NO_CONVERSION); + posReference++; + ALICEVISION_LOG_INFO("processing input region " << posReference << "/" << sfmData.getViews().size()); + + IndexT viewReference = vIterator.first; + std::list overlaps; + if (!panoramaMap->getOverlaps(overlaps, viewReference)) + { + ALICEVISION_LOG_ERROR("Problem analyzing neighboorhood"); + return EXIT_FAILURE; + } + overlaps.push_back(viewReference); - // Load mask - const std::string maskPath = (fs::path(warpingFolder) / (std::to_string(viewCurrent) + "_mask.exr")).string(); - ALICEVISION_LOG_TRACE("Load mask with path " << maskPath); - image::Image mask; - image::readImage(maskPath, mask, image::EImageColorSpace::NO_CONVERSION); + BoundingBox referenceBoundingBox; + if (!panoramaMap->getEnhancedBoundingBox(referenceBoundingBox, viewReference)) + { + ALICEVISION_LOG_ERROR("Problem getting reference bounding box"); + return EXIT_FAILURE; + } - // Retrieve position of image in panorama - oiio::ParamValueList metadata = image::readImageMetadata(imagePath); - const int offsetX = metadata.find("AliceVision:offsetX")->get_int(); - const int offsetY = metadata.find("AliceVision:offsetY")->get_int(); + std::unique_ptr compositer; + compositer = std::unique_ptr(new LaplacianCompositer(referenceBoundingBox.width, referenceBoundingBox.height, panoramaMap->getScale())); - image::Image weights; - /*const std::string weightsPath = (fs::path(warpingFolder) / (std::to_string(viewCurrent) + "_weight.exr")).string(); - ALICEVISION_LOG_TRACE("Load weights with path " << weightsPath); - image::readImage(weightsPath, weights, image::EImageColorSpace::NO_CONVERSION);*/ - weights = image::Image(mask.Width(), mask.Height()); - if (!getMaskFromLabels(weights, labels, viewCurrent, offsetX - referenceBoundingBox.left, offsetY - referenceBoundingBox.top)) + if (!compositer->initialize()) { - ALICEVISION_LOG_ERROR("Error estimating seams image"); - return EXIT_FAILURE; + ALICEVISION_LOG_ERROR("Error initializing panorama"); + return false; } - if (!compositer->append(source, mask, weights, offsetX - referenceBoundingBox.left, offsetY - referenceBoundingBox.top)) + image::Image labels; + if (!computeWTALabels(labels, overlaps, warpingFolder, referenceBoundingBox)) { - ALICEVISION_LOG_INFO("Error in compositer append"); - return EXIT_FAILURE; - } + ALICEVISION_LOG_ERROR("Error estimating panorama labels for this input"); + return false; + } - pos++; - } + int posCurrent = 0; + for (IndexT viewCurrent : overlaps) + { + posCurrent++; + ALICEVISION_LOG_INFO("Processing input " << posCurrent << "/" << overlaps.size()); + + // Load image and convert it to linear colorspace + const std::string imagePath = (fs::path(warpingFolder) / (std::to_string(viewCurrent) + ".exr")).string(); + ALICEVISION_LOG_TRACE("Load image with path " << imagePath); + image::Image source; + image::readImage(imagePath, source, image::EImageColorSpace::NO_CONVERSION); + + // Load mask + const std::string maskPath = (fs::path(warpingFolder) / (std::to_string(viewCurrent) + "_mask.exr")).string(); + ALICEVISION_LOG_TRACE("Load mask with path " << maskPath); + image::Image mask; + image::readImage(maskPath, mask, image::EImageColorSpace::NO_CONVERSION); + + // Retrieve position of image in panorama + oiio::ParamValueList metadata = image::readImageMetadata(imagePath); + const int offsetX = metadata.find("AliceVision:offsetX")->get_int(); + const int offsetY = metadata.find("AliceVision:offsetY")->get_int(); + + image::Image weights; + /*const std::string weightsPath = (fs::path(warpingFolder) / (std::to_string(viewCurrent) + "_weight.exr")).string(); + ALICEVISION_LOG_TRACE("Load weights with path " << weightsPath); + image::readImage(weightsPath, weights, image::EImageColorSpace::NO_CONVERSION);*/ + weights = image::Image(mask.Width(), mask.Height()); + if (!getMaskFromLabels(weights, labels, viewCurrent, offsetX - referenceBoundingBox.left, offsetY - referenceBoundingBox.top)) + { + ALICEVISION_LOG_ERROR("Error estimating seams image"); + return EXIT_FAILURE; + } + + if (!compositer->append(source, mask, weights, offsetX - referenceBoundingBox.left, offsetY - referenceBoundingBox.top)) + { + ALICEVISION_LOG_INFO("Error in compositer append"); + return EXIT_FAILURE; + } + } - if (!compositer->terminate()) - { - ALICEVISION_LOG_ERROR("Error terminating panorama"); - } + if (!compositer->terminate()) + { + ALICEVISION_LOG_ERROR("Error terminating panorama"); + } - if (!compositer->save("/home/servantf/test.exr", image::EStorageDataType::HalfFinite)) - { - ALICEVISION_LOG_ERROR("Error terminating panorama"); + const std::string viewIdStr = std::to_string(viewReference); + const std::string outputFilePath = (fs::path(outputFolder) / (viewIdStr + ".exr")).string(); + if (!compositer->save(outputFilePath, storageDataType)) + { + ALICEVISION_LOG_ERROR("Error terminating panorama"); + } } return EXIT_SUCCESS; From b881b2553db4dfcfd5a3d6e132cd9a46c139cc55 Mon Sep 17 00:00:00 2001 From: Fabien Date: Tue, 15 Dec 2020 14:02:48 +0100 Subject: [PATCH 47/79] [panorama] remove some bugs for large warping --- src/aliceVision/panorama/laplacianPyramid.cpp | 24 +--- src/aliceVision/panorama/laplacianPyramid.hpp | 1 - src/aliceVision/panorama/panoramaMap.cpp | 5 +- src/aliceVision/panorama/panoramaMap.hpp | 14 +++ src/aliceVision/panorama/warper.cpp | 9 +- src/aliceVision/panorama/warper.hpp | 3 +- .../pipeline/main_panoramaCompositing.cpp | 109 +++++++++++++++--- .../pipeline/main_panoramaWarping.cpp | 39 ++++--- 8 files changed, 148 insertions(+), 56 deletions(-) diff --git a/src/aliceVision/panorama/laplacianPyramid.cpp b/src/aliceVision/panorama/laplacianPyramid.cpp index ebd33738aa..c2e811e042 100644 --- a/src/aliceVision/panorama/laplacianPyramid.cpp +++ b/src/aliceVision/panorama/laplacianPyramid.cpp @@ -26,22 +26,13 @@ bool LaplacianPyramid::initialize() /*Prepare pyramid*/ for(int lvl = 0; lvl < _maxLevels; lvl++) { - //If the level width is odd, it is a problem. - //Let update this level to an even size. we'll manage it later - //Store the real size to know we updated it - _realWidths.push_back(width); - if (width % 2) - { - width++; - } - image::Image color(width, height, true, image::RGBfColor(0.0f, 0.0f, 0.0f)); image::Image weights(width, height, true, 0.0f); _levels.push_back(color); _weights.push_back(weights); - width = width / 2; + width = int(ceil(float(width) / 2.0f)); height = int(ceil(float(height) / 2.0f)); } @@ -67,18 +58,9 @@ bool LaplacianPyramid::augment(size_t newMaxLevels) int width = _levels[_levels.size() - 1].Width(); int height = _levels[_levels.size() - 1].Height(); - int nextWidth = width / 2; + int nextWidth = int(ceil(float(width) / 2.0f)); int nextHeight = int(ceil(float(height) / 2.0f)); - //If the level width is odd, it is a problem. - //Let update this level to an even size. we'll manage it later - //Store the real size to know we updated it - _realWidths.push_back(nextWidth); - if (nextWidth % 2) - { - nextWidth++; - } - image::Image pyramidImage(nextWidth, nextHeight, true, image::RGBfColor(0.0f)); image::Image pyramidWeights(nextWidth, nextHeight, true, 0.0f); @@ -372,7 +354,7 @@ bool LaplacianPyramid::rebuild(image::Image& output) buf2(y, x) *= 4.0f; } } - + if (!addition(_levels[currentLevel], _levels[currentLevel], buf2)) { return false; diff --git a/src/aliceVision/panorama/laplacianPyramid.hpp b/src/aliceVision/panorama/laplacianPyramid.hpp index ad51d0dfe4..a167d1522d 100644 --- a/src/aliceVision/panorama/laplacianPyramid.hpp +++ b/src/aliceVision/panorama/laplacianPyramid.hpp @@ -44,7 +44,6 @@ class LaplacianPyramid std::vector> _levels; std::vector> _weights; - std::vector _realWidths; std::vector _inputInfos; }; diff --git a/src/aliceVision/panorama/panoramaMap.cpp b/src/aliceVision/panorama/panoramaMap.cpp index 8d02e2ff28..cc99d2d98a 100644 --- a/src/aliceVision/panorama/panoramaMap.cpp +++ b/src/aliceVision/panorama/panoramaMap.cpp @@ -15,6 +15,7 @@ bool PanoramaMap::append(IndexT index, const BoundingBox & box) BoundingBox scaledWithBorders = scaled.dilate(_borderSize); _map[index] = scaledWithBorders.multiply(maxFactor); + _mapRaw[index] = box; return true; } @@ -40,7 +41,7 @@ bool PanoramaMap::intersect(const BoundingBox & box1, const BoundingBox & box2) return true; } - if (!box1.intersectionWith(otherBbox).isEmpty()) + if (!box1.intersectionWith(otherBboxLoop).isEmpty()) { /*BoundingBox sbox1 = box1.divide(_scale); BoundingBox sotherBbox = otherBboxLoop.divide(_scale); @@ -85,6 +86,8 @@ bool PanoramaMap::getOverlaps(std::list & overlaps, IndexT reference) continue; } + std::cout << it.first << " " << it.second << " " << bbref << std::endl; + if (intersect(bbref, it.second)) { overlaps.push_back(it.first); diff --git a/src/aliceVision/panorama/panoramaMap.hpp b/src/aliceVision/panorama/panoramaMap.hpp index 4065d48d04..aedf86f978 100644 --- a/src/aliceVision/panorama/panoramaMap.hpp +++ b/src/aliceVision/panorama/panoramaMap.hpp @@ -53,11 +53,25 @@ class PanoramaMap return true; } + bool getBoundingBox(BoundingBox & bb, const IndexT & id) + { + if (_mapRaw.find(id) == _mapRaw.end()) + { + return false; + } + + bb = _mapRaw[id]; + + return true; + } + + private: bool intersect(const BoundingBox & box1, const BoundingBox & box2) const; private: std::map _map; + std::map _mapRaw; int _panoramaWidth; int _panoramaHeight; diff --git a/src/aliceVision/panorama/warper.cpp b/src/aliceVision/panorama/warper.cpp index 7a5d2a0fe8..bd6b775eb1 100644 --- a/src/aliceVision/panorama/warper.cpp +++ b/src/aliceVision/panorama/warper.cpp @@ -1,4 +1,5 @@ #include "warper.hpp" +#include namespace aliceVision { @@ -46,7 +47,7 @@ bool Warper::warp(const CoordinatesMap& map, const aliceVision::image::Image HALF_MAX) _color(i, j).r() = HALF_MAX; + if (_color(i, j).g() > HALF_MAX) _color(i, j).g() = HALF_MAX; + if (_color(i, j).b() > HALF_MAX) _color(i, j).b() = HALF_MAX; + } } } diff --git a/src/aliceVision/panorama/warper.hpp b/src/aliceVision/panorama/warper.hpp index 7039e90a4b..5a479286fa 100644 --- a/src/aliceVision/panorama/warper.hpp +++ b/src/aliceVision/panorama/warper.hpp @@ -3,6 +3,7 @@ #include "coordinatesMap.hpp" #include "gaussian.hpp" + namespace aliceVision { @@ -30,7 +31,7 @@ class Warper class GaussianWarper : public Warper { public: - virtual bool warp(const CoordinatesMap& map, const GaussianPyramidNoMask& pyramid); + virtual bool warp(const CoordinatesMap& map, const GaussianPyramidNoMask& pyramid, bool clamp); }; } // namespace aliceVision \ No newline at end of file diff --git a/src/software/pipeline/main_panoramaCompositing.cpp b/src/software/pipeline/main_panoramaCompositing.cpp index 84877c0139..cbb982f6a2 100644 --- a/src/software/pipeline/main_panoramaCompositing.cpp +++ b/src/software/pipeline/main_panoramaCompositing.cpp @@ -113,12 +113,13 @@ std::unique_ptr buildMap(const sfmData::SfMData & sfmData, const st return ret; } -bool computeWTALabels(image::Image & labels, std::list & views, const std::string & inputPath, const BoundingBox & boundingBox) +bool computeWTALabels(image::Image & labels, std::list & views, const std::string & inputPath, const BoundingBox & boundingBox, int panoramaWidth) { ALICEVISION_LOG_INFO("Estimating initial labels for panorama"); WTASeams seams(boundingBox.width, boundingBox.height); + for (const IndexT & currentId : views) { // Load mask @@ -137,10 +138,43 @@ bool computeWTALabels(image::Image & labels, std::list & views, ALICEVISION_LOG_TRACE("Load weights with path " << weightsPath); image::Image weights; image::readImage(weightsPath, weights, image::EImageColorSpace::NO_CONVERSION); - - if (!seams.append(mask, weights, currentId, offsetX - boundingBox.left, offsetY - boundingBox.top)) + + BoundingBox currentBb; + currentBb.top = offsetY; + currentBb.width = mask.Width(); + currentBb.height = mask.Height(); + currentBb.left = offsetX; + if (!boundingBox.intersectionWith(currentBb).isEmpty()) { - return false; + if (!seams.append(mask, weights, currentId, currentBb.left - boundingBox.left, offsetY - boundingBox.top)) + { + return false; + } + } + + + currentBb.left = offsetX - panoramaWidth; + currentBb.top = offsetY; + currentBb.width = mask.Width(); + currentBb.height = mask.Height(); + if (!boundingBox.intersectionWith(currentBb).isEmpty()) + { + if (!seams.append(mask, weights, currentId, currentBb.left - boundingBox.left, offsetY - boundingBox.top)) + { + return false; + } + } + + currentBb.left = offsetX + panoramaWidth; + currentBb.top = offsetY; + currentBb.width = mask.Width(); + currentBb.height = mask.Height(); + if (!boundingBox.intersectionWith(currentBb).isEmpty()) + { + if (!seams.append(mask, weights, currentId, currentBb.left - boundingBox.left, offsetY - boundingBox.top)) + { + return false; + } } } @@ -254,6 +288,7 @@ int aliceVision_main(int argc, char** argv) rangeSize = int(viewsCount); } + const size_t borderSize = 2; std::unique_ptr panoramaMap = buildMap(sfmData, warpingFolder, borderSize); if (viewsCount == 0) @@ -262,13 +297,22 @@ int aliceVision_main(int argc, char** argv) return EXIT_FAILURE; } - int posReference = 0; + std::vector> views; for (auto & vIterator : sfmData.getViews()) { - posReference++; - ALICEVISION_LOG_INFO("processing input region " << posReference << "/" << sfmData.getViews().size()); + views.push_back(vIterator.second); + } + + for (std::size_t posReference = std::size_t(rangeStart); posReference < std::size_t(rangeStart + rangeSize); ++posReference) + { + ALICEVISION_LOG_INFO("processing input region " << posReference + 1 << "/" << views.size()); - IndexT viewReference = vIterator.first; + IndexT viewReference = views[posReference]->getViewId(); + /*if (viewReference != 347875201) + { + continue; + }*/ + std::list overlaps; if (!panoramaMap->getOverlaps(overlaps, viewReference)) { @@ -278,12 +322,19 @@ int aliceVision_main(int argc, char** argv) overlaps.push_back(viewReference); BoundingBox referenceBoundingBox; - if (!panoramaMap->getEnhancedBoundingBox(referenceBoundingBox, viewReference)) + if (!panoramaMap->getBoundingBox(referenceBoundingBox, viewReference)) { ALICEVISION_LOG_ERROR("Problem getting reference bounding box"); return EXIT_FAILURE; } + bool shifted = false; + if (referenceBoundingBox.getRight() >= panoramaMap->getWidth()) + { + referenceBoundingBox.left -= panoramaMap->getWidth(); + shifted = true; + } + std::unique_ptr compositer; compositer = std::unique_ptr(new LaplacianCompositer(referenceBoundingBox.width, referenceBoundingBox.height, panoramaMap->getScale())); @@ -294,17 +345,24 @@ int aliceVision_main(int argc, char** argv) } image::Image labels; - if (!computeWTALabels(labels, overlaps, warpingFolder, referenceBoundingBox)) + if (!computeWTALabels(labels, overlaps, warpingFolder, referenceBoundingBox, panoramaMap->getWidth())) { ALICEVISION_LOG_ERROR("Error estimating panorama labels for this input"); return false; } - + + { + const std::string viewIdStr = std::to_string(viewReference); + const std::string outputFilePath = (fs::path(outputFolder) / (viewIdStr + "_label.exr")).string(); + image::writeImage(outputFilePath, labels, image::EImageColorSpace::NO_CONVERSION); + } + int posCurrent = 0; for (IndexT viewCurrent : overlaps) { posCurrent++; ALICEVISION_LOG_INFO("Processing input " << posCurrent << "/" << overlaps.size()); + // Load image and convert it to linear colorspace const std::string imagePath = (fs::path(warpingFolder) / (std::to_string(viewCurrent) + ".exr")).string(); @@ -320,14 +378,20 @@ int aliceVision_main(int argc, char** argv) // Retrieve position of image in panorama oiio::ParamValueList metadata = image::readImageMetadata(imagePath); - const int offsetX = metadata.find("AliceVision:offsetX")->get_int(); - const int offsetY = metadata.find("AliceVision:offsetY")->get_int(); + int offsetX = metadata.find("AliceVision:offsetX")->get_int(); + int offsetY = metadata.find("AliceVision:offsetY")->get_int(); + + if (offsetX > referenceBoundingBox.getRight()) + { + offsetX -= panoramaMap->getWidth(); + } image::Image weights; /*const std::string weightsPath = (fs::path(warpingFolder) / (std::to_string(viewCurrent) + "_weight.exr")).string(); ALICEVISION_LOG_TRACE("Load weights with path " << weightsPath); image::readImage(weightsPath, weights, image::EImageColorSpace::NO_CONVERSION);*/ weights = image::Image(mask.Width(), mask.Height()); + if (!getMaskFromLabels(weights, labels, viewCurrent, offsetX - referenceBoundingBox.left, offsetY - referenceBoundingBox.top)) { ALICEVISION_LOG_ERROR("Error estimating seams image"); @@ -338,9 +402,26 @@ int aliceVision_main(int argc, char** argv) { ALICEVISION_LOG_INFO("Error in compositer append"); return EXIT_FAILURE; - } + } + + if (mask.Width() >= panoramaMap->getWidth()) + { + if (!getMaskFromLabels(weights, labels, viewCurrent, offsetX - panoramaMap->getWidth() - referenceBoundingBox.left, offsetY - referenceBoundingBox.top)) + { + ALICEVISION_LOG_ERROR("Error estimating seams image"); + return EXIT_FAILURE; + } + + if (!compositer->append(source, mask, weights, offsetX - panoramaMap->getWidth() - referenceBoundingBox.left, offsetY - referenceBoundingBox.top)) + { + ALICEVISION_LOG_INFO("Error in compositer append"); + return EXIT_FAILURE; + } + } } + + std::cout << "terminate" << std::endl; if (!compositer->terminate()) { ALICEVISION_LOG_ERROR("Error terminating panorama"); diff --git a/src/software/pipeline/main_panoramaWarping.cpp b/src/software/pipeline/main_panoramaWarping.cpp index 06722de793..41c0a52759 100644 --- a/src/software/pipeline/main_panoramaWarping.cpp +++ b/src/software/pipeline/main_panoramaWarping.cpp @@ -170,10 +170,15 @@ int aliceVision_main(int argc, char** argv) // Set verbose level given command line system::Logger::get()->setLogLevel(verboseLevel); + bool clampHalf = false; oiio::TypeDesc typeColor = oiio::TypeDesc::FLOAT; if (storageDataType == image::EStorageDataType::Half || storageDataType == image::EStorageDataType::HalfFinite) { typeColor = oiio::TypeDesc::HALF; + if (storageDataType == image::EStorageDataType::HalfFinite) + { + clampHalf = true; + } } // Load information about inputs @@ -229,24 +234,24 @@ int aliceVision_main(int argc, char** argv) // If panorama width is undefined, estimate it if(panoramaSize.first <= 0) { - float ratioUpscale = clamp(float(percentUpscale) / 100.0f, 0.0f, 1.0f); + float ratioUpscale = clamp(float(percentUpscale) / 100.0f, 0.0f, 1.0f); - std::pair optimalPanoramaSize; - if(computeOptimalPanoramaSize(optimalPanoramaSize, sfmData, ratioUpscale)) - { - panoramaSize = optimalPanoramaSize; - } - else - { - ALICEVISION_LOG_INFO("Impossible to compute an optimal panorama size"); - return EXIT_FAILURE; - } + std::pair optimalPanoramaSize; + if(computeOptimalPanoramaSize(optimalPanoramaSize, sfmData, ratioUpscale)) + { + panoramaSize = optimalPanoramaSize; + } + else + { + ALICEVISION_LOG_INFO("Impossible to compute an optimal panorama size"); + return EXIT_FAILURE; + } - if (maxPanoramaWidth != 0 && panoramaSize.first > maxPanoramaWidth) - { - ALICEVISION_LOG_INFO("The optimal size of the panorama exceeds the maximum size (estimated width: " << panoramaSize.first << ", max width: " << maxPanoramaWidth << ")."); - panoramaSize.first = maxPanoramaWidth; - } + if (maxPanoramaWidth != 0 && panoramaSize.first > maxPanoramaWidth) + { + ALICEVISION_LOG_INFO("The optimal size of the panorama exceeds the maximum size (estimated width: " << panoramaSize.first << ", max width: " << maxPanoramaWidth << ")."); + panoramaSize.first = maxPanoramaWidth; + } } // Make sure panorama size has required size properties @@ -437,7 +442,7 @@ int aliceVision_main(int argc, char** argv) // Warp image GaussianWarper warper; - if (!warper.warp(map, pyramid)) { + if (!warper.warp(map, pyramid, clampHalf)) { continue; } From d88f7eb4e41f08a41822725525f37acdf66d39d4 Mon Sep 17 00:00:00 2001 From: Fabien Date: Tue, 15 Dec 2020 14:16:16 +0100 Subject: [PATCH 48/79] [panorama] fix some special case --- src/aliceVision/panorama/panoramaMap.cpp | 2 -- src/software/pipeline/main_panoramaCompositing.cpp | 7 +++---- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/src/aliceVision/panorama/panoramaMap.cpp b/src/aliceVision/panorama/panoramaMap.cpp index cc99d2d98a..a3d312648a 100644 --- a/src/aliceVision/panorama/panoramaMap.cpp +++ b/src/aliceVision/panorama/panoramaMap.cpp @@ -86,8 +86,6 @@ bool PanoramaMap::getOverlaps(std::list & overlaps, IndexT reference) continue; } - std::cout << it.first << " " << it.second << " " << bbref << std::endl; - if (intersect(bbref, it.second)) { overlaps.push_back(it.first); diff --git a/src/software/pipeline/main_panoramaCompositing.cpp b/src/software/pipeline/main_panoramaCompositing.cpp index cbb982f6a2..b0d05cbed5 100644 --- a/src/software/pipeline/main_panoramaCompositing.cpp +++ b/src/software/pipeline/main_panoramaCompositing.cpp @@ -308,7 +308,7 @@ int aliceVision_main(int argc, char** argv) ALICEVISION_LOG_INFO("processing input region " << posReference + 1 << "/" << views.size()); IndexT viewReference = views[posReference]->getViewId(); - /*if (viewReference != 347875201) + /*if (viewReference != 672262770) { continue; }*/ @@ -329,7 +329,7 @@ int aliceVision_main(int argc, char** argv) } bool shifted = false; - if (referenceBoundingBox.getRight() >= panoramaMap->getWidth()) + if (referenceBoundingBox.getRight() >= panoramaMap->getWidth() && referenceBoundingBox.width < panoramaMap->getWidth()) { referenceBoundingBox.left -= panoramaMap->getWidth(); shifted = true; @@ -363,7 +363,6 @@ int aliceVision_main(int argc, char** argv) posCurrent++; ALICEVISION_LOG_INFO("Processing input " << posCurrent << "/" << overlaps.size()); - // Load image and convert it to linear colorspace const std::string imagePath = (fs::path(warpingFolder) / (std::to_string(viewCurrent) + ".exr")).string(); ALICEVISION_LOG_TRACE("Load image with path " << imagePath); @@ -404,7 +403,7 @@ int aliceVision_main(int argc, char** argv) return EXIT_FAILURE; } - if (mask.Width() >= panoramaMap->getWidth()) + if (mask.Width() >= panoramaMap->getWidth() || referenceBoundingBox.width >= panoramaMap->getWidth()) { if (!getMaskFromLabels(weights, labels, viewCurrent, offsetX - panoramaMap->getWidth() - referenceBoundingBox.left, offsetY - referenceBoundingBox.top)) { From d7544bfccebb5e89f10d5959f68a8d35df0082b5 Mon Sep 17 00:00:00 2001 From: Fabien Date: Thu, 17 Dec 2020 10:33:50 +0100 Subject: [PATCH 49/79] [panorama] make sure only the valid part of an input is processed --- src/aliceVision/panorama/boundingBox.hpp | 34 +++++ src/aliceVision/panorama/panoramaMap.cpp | 107 ++++++++------ src/aliceVision/panorama/panoramaMap.hpp | 20 +-- .../pipeline/main_panoramaCompositing.cpp | 138 ++++++++---------- 4 files changed, 164 insertions(+), 135 deletions(-) diff --git a/src/aliceVision/panorama/boundingBox.hpp b/src/aliceVision/panorama/boundingBox.hpp index dd30e9fb8e..b71e057c4f 100644 --- a/src/aliceVision/panorama/boundingBox.hpp +++ b/src/aliceVision/panorama/boundingBox.hpp @@ -210,6 +210,40 @@ struct BoundingBox return b; } + + BoundingBox limitInside(const BoundingBox & other) const + { + BoundingBox b; + + b.left = left; + if (b.left < other.left) + { + b.left = other.left; + } + + b.top = top; + if (b.top < other.top) + { + b.top = other.top; + } + + int nright = getRight(); + if (nright > other.getRight()) + { + nright = other.getRight(); + } + + int nbottom = getBottom(); + if (nbottom > other.getBottom()) + { + nbottom = other.getBottom(); + } + + b.width = nright - b.left + 1; + b.height = nbottom - b.top + 1; + + return b; + } }; std::ostream& operator<<(std::ostream& os, const BoundingBox& in); diff --git a/src/aliceVision/panorama/panoramaMap.cpp b/src/aliceVision/panorama/panoramaMap.cpp index a3d312648a..072e69e9fc 100644 --- a/src/aliceVision/panorama/panoramaMap.cpp +++ b/src/aliceVision/panorama/panoramaMap.cpp @@ -8,76 +8,57 @@ namespace aliceVision bool PanoramaMap::append(IndexT index, const BoundingBox & box) { - int maxFactor = pow(2, _scale); + BoundingBox bbox = box; - BoundingBox scaled = box.divide(maxFactor); - - BoundingBox scaledWithBorders = scaled.dilate(_borderSize); + // if the reference bounding box is looping on the right of the panorama, + // make it loop on the left instead to reduce the number of further possibilities + if (bbox.getRight() >= _panoramaWidth && bbox.width < _panoramaWidth) + { + bbox.left -= _panoramaWidth; + } - _map[index] = scaledWithBorders.multiply(maxFactor); - _mapRaw[index] = box; + _map[index] = bbox; return true; } bool PanoramaMap::intersect(const BoundingBox & box1, const BoundingBox & box2) const { - BoundingBox otherBbox = box2; - BoundingBox otherBboxLoop = box2; + BoundingBox extentedBox1 = box1.divide(_scale).dilate(_borderSize).multiply(_scale); + BoundingBox extentedBox2 = box2.divide(_scale).dilate(_borderSize).multiply(_scale); + + BoundingBox otherBbox = extentedBox2; + BoundingBox otherBboxLoop = extentedBox2; otherBboxLoop.left = otherBbox.left - _panoramaWidth; - BoundingBox otherBboxLoopRight = box2; + BoundingBox otherBboxLoopRight = extentedBox2; otherBboxLoopRight.left = otherBbox.left + _panoramaWidth; - if (!box1.intersectionWith(otherBbox).isEmpty()) + if (!extentedBox1.intersectionWith(otherBbox).isEmpty()) { - /*BoundingBox sbox1 = box1.divide(_scale); - BoundingBox sotherBbox = otherBbox.divide(_scale); - BoundingBox bb = sbox1.intersectionWith(sotherBbox); - BoundingBox bbb = bb.multiply(_scale); - - std::cout << bbb << std::endl; - std::cout << "(" << box2 << ")" << std::endl;*/ - return true; } - if (!box1.intersectionWith(otherBboxLoop).isEmpty()) + if (!extentedBox1.intersectionWith(otherBboxLoop).isEmpty()) { - /*BoundingBox sbox1 = box1.divide(_scale); - BoundingBox sotherBbox = otherBboxLoop.divide(_scale); - BoundingBox bb = sbox1.intersectionWith(sotherBbox); - BoundingBox bbb = bb.multiply(_scale); - - std::cout << bbb << std::endl; - std::cout << "(" << box2 << ")" << std::endl;*/ - return true; } - if (!box1.intersectionWith(otherBboxLoopRight).isEmpty()) + if (!extentedBox1.intersectionWith(otherBboxLoopRight).isEmpty()) { - /*BoundingBox sbox1 = box1.divide(_scale); - BoundingBox sotherBbox = otherBboxLoopRight.divide(_scale); - BoundingBox bb = sbox1.intersectionWith(sotherBbox); - BoundingBox bbb = bb.multiply(_scale); - - std::cout << bbb << std::endl; - std::cout << "(" << box2 << ")" << std::endl;*/ - return true; } return false; } -bool PanoramaMap::getOverlaps(std::list & overlaps, IndexT reference) +bool PanoramaMap::getOverlaps(std::list & overlaps, IndexT reference) const { if (_map.find(reference) == _map.end()) { return false; } - BoundingBox bbref = _map[reference]; + BoundingBox bbref = _map.at(reference); for (auto it : _map) { @@ -95,15 +76,57 @@ bool PanoramaMap::getOverlaps(std::list & overlaps, IndexT reference) return true; } -bool PanoramaMap::getOverlaps(std::list & overlaps, BoundingBox bbref) +bool PanoramaMap::getIntersectionsList(std::vector & intersections, std::vector & currentBoundingBoxes, const IndexT & referenceIndex, const IndexT & otherIndex) const { + BoundingBox referenceBoundingBox = _map.at(referenceIndex); + BoundingBox referenceBoundingBoxReduced = referenceBoundingBox.divide(_scale).dilate(_borderSize); + + BoundingBox otherBoundingBox = _map.at(otherIndex); - for (auto it : _map) + // Base compare { - if (intersect(bbref, it.second)) + BoundingBox otherBoundingBoxReduced = otherBoundingBox.divide(_scale).dilate(_borderSize); + BoundingBox intersectionSmall = referenceBoundingBoxReduced.intersectionWith(otherBoundingBoxReduced); + if (!intersectionSmall.isEmpty()) { - overlaps.push_back(it.first); + currentBoundingBoxes.push_back(otherBoundingBox); + + BoundingBox intersection = intersectionSmall.multiply(_scale); + intersection = intersection.limitInside(otherBoundingBox); + intersections.push_back(intersection); + } + } + + // Shift to check loop + { + BoundingBox otherBoundingBoxLoop = otherBoundingBox; + otherBoundingBoxLoop.left -= _panoramaWidth; + BoundingBox otherBoundingBoxReduced = otherBoundingBoxLoop.divide(_scale).dilate(_borderSize); + BoundingBox intersectionSmall = referenceBoundingBoxReduced.intersectionWith(otherBoundingBoxReduced); + if (!intersectionSmall.isEmpty()) + { + currentBoundingBoxes.push_back(otherBoundingBoxLoop); + + BoundingBox intersection = intersectionSmall.multiply(_scale); + intersection = intersection.limitInside(otherBoundingBoxLoop); + intersections.push_back(intersection); + } + } + + // Shift to check loop + { + BoundingBox otherBoundingBoxLoop = otherBoundingBox; + otherBoundingBoxLoop.left += _panoramaWidth; + BoundingBox otherBoundingBoxReduced = otherBoundingBoxLoop.divide(_scale).dilate(_borderSize); + BoundingBox intersectionSmall = referenceBoundingBoxReduced.intersectionWith(otherBoundingBoxReduced); + if (!intersectionSmall.isEmpty()) + { + currentBoundingBoxes.push_back(otherBoundingBoxLoop); + + BoundingBox intersection = intersectionSmall.multiply(_scale); + intersection = intersection.limitInside(otherBoundingBoxLoop); + intersections.push_back(intersection); } } diff --git a/src/aliceVision/panorama/panoramaMap.hpp b/src/aliceVision/panorama/panoramaMap.hpp index aedf86f978..6649e9beb0 100644 --- a/src/aliceVision/panorama/panoramaMap.hpp +++ b/src/aliceVision/panorama/panoramaMap.hpp @@ -23,8 +23,7 @@ class PanoramaMap bool append(IndexT index, const BoundingBox & box); - bool getOverlaps(std::list & overlaps, IndexT reference); - bool getOverlaps(std::list & overlaps, BoundingBox bbref); + bool getOverlaps(std::list & overlaps, IndexT reference) const; size_t getWidth() const { @@ -41,29 +40,19 @@ class PanoramaMap return _scale; } - bool getEnhancedBoundingBox(BoundingBox & bb, const IndexT & id) + bool getBoundingBox(BoundingBox & bb, const IndexT & id) const { if (_map.find(id) == _map.end()) { return false; } - bb = _map[id]; + bb = _map.at(id); return true; } - bool getBoundingBox(BoundingBox & bb, const IndexT & id) - { - if (_mapRaw.find(id) == _mapRaw.end()) - { - return false; - } - - bb = _mapRaw[id]; - - return true; - } + bool getIntersectionsList(std::vector & intersections, std::vector & currentBoundingBoxes, const IndexT & referenceIndex, const IndexT & otherIndex) const; private: @@ -71,7 +60,6 @@ class PanoramaMap private: std::map _map; - std::map _mapRaw; int _panoramaWidth; int _panoramaHeight; diff --git a/src/software/pipeline/main_panoramaCompositing.cpp b/src/software/pipeline/main_panoramaCompositing.cpp index b0d05cbed5..d11d2d296f 100644 --- a/src/software/pipeline/main_panoramaCompositing.cpp +++ b/src/software/pipeline/main_panoramaCompositing.cpp @@ -113,12 +113,18 @@ std::unique_ptr buildMap(const sfmData::SfMData & sfmData, const st return ret; } -bool computeWTALabels(image::Image & labels, std::list & views, const std::string & inputPath, const BoundingBox & boundingBox, int panoramaWidth) +bool computeWTALabels(image::Image & labels, std::list & views, const std::string & inputPath, const PanoramaMap & map, const IndexT & referenceIndex) { ALICEVISION_LOG_INFO("Estimating initial labels for panorama"); - WTASeams seams(boundingBox.width, boundingBox.height); + BoundingBox referenceBoundingBox; + + if (!map.getBoundingBox(referenceBoundingBox, referenceIndex)) + { + return false; + } + WTASeams seams(referenceBoundingBox.width, referenceBoundingBox.height); for (const IndexT & currentId : views) { @@ -139,39 +145,17 @@ bool computeWTALabels(image::Image & labels, std::list & views, image::Image weights; image::readImage(weightsPath, weights, image::EImageColorSpace::NO_CONVERSION); - BoundingBox currentBb; - currentBb.top = offsetY; - currentBb.width = mask.Width(); - currentBb.height = mask.Height(); - currentBb.left = offsetX; - if (!boundingBox.intersectionWith(currentBb).isEmpty()) - { - if (!seams.append(mask, weights, currentId, currentBb.left - boundingBox.left, offsetY - boundingBox.top)) - { - return false; - } - } - - - currentBb.left = offsetX - panoramaWidth; - currentBb.top = offsetY; - currentBb.width = mask.Width(); - currentBb.height = mask.Height(); - if (!boundingBox.intersectionWith(currentBb).isEmpty()) + + std::vector intersections; + std::vector currentBoundingBoxes; + if (!map.getIntersectionsList(intersections, currentBoundingBoxes, referenceIndex, currentId)) { - if (!seams.append(mask, weights, currentId, currentBb.left - boundingBox.left, offsetY - boundingBox.top)) - { - return false; - } + continue; } - - currentBb.left = offsetX + panoramaWidth; - currentBb.top = offsetY; - currentBb.width = mask.Width(); - currentBb.height = mask.Height(); - if (!boundingBox.intersectionWith(currentBb).isEmpty()) + + for (const BoundingBox & bbox : currentBoundingBoxes) { - if (!seams.append(mask, weights, currentId, currentBb.left - boundingBox.left, offsetY - boundingBox.top)) + if (!seams.append(mask, weights, currentId, bbox.left - referenceBoundingBox.left, bbox.top - referenceBoundingBox.top)) { return false; } @@ -288,7 +272,8 @@ int aliceVision_main(int argc, char** argv) rangeSize = int(viewsCount); } - + // Build the map of inputs in the final panorama + // This is mostly meant to compute overlaps between inputs const size_t borderSize = 2; std::unique_ptr panoramaMap = buildMap(sfmData, warpingFolder, borderSize); if (viewsCount == 0) @@ -297,6 +282,7 @@ int aliceVision_main(int argc, char** argv) return EXIT_FAILURE; } + // Build a list of views std::vector> views; for (auto & vIterator : sfmData.getViews()) { @@ -308,19 +294,19 @@ int aliceVision_main(int argc, char** argv) ALICEVISION_LOG_INFO("processing input region " << posReference + 1 << "/" << views.size()); IndexT viewReference = views[posReference]->getViewId(); - /*if (viewReference != 672262770) - { - continue; - }*/ - + //if (viewReference != 137597937) continue; + // Get the list of input which should be processed for this reference view bounding box std::list overlaps; if (!panoramaMap->getOverlaps(overlaps, viewReference)) { ALICEVISION_LOG_ERROR("Problem analyzing neighboorhood"); return EXIT_FAILURE; } + + // Add the current input also for simpler processing overlaps.push_back(viewReference); + // Get the input bounding box to define the ROI BoundingBox referenceBoundingBox; if (!panoramaMap->getBoundingBox(referenceBoundingBox, viewReference)) { @@ -328,16 +314,8 @@ int aliceVision_main(int argc, char** argv) return EXIT_FAILURE; } - bool shifted = false; - if (referenceBoundingBox.getRight() >= panoramaMap->getWidth() && referenceBoundingBox.width < panoramaMap->getWidth()) - { - referenceBoundingBox.left -= panoramaMap->getWidth(); - shifted = true; - } - std::unique_ptr compositer; compositer = std::unique_ptr(new LaplacianCompositer(referenceBoundingBox.width, referenceBoundingBox.height, panoramaMap->getScale())); - if (!compositer->initialize()) { ALICEVISION_LOG_ERROR("Error initializing panorama"); @@ -345,17 +323,17 @@ int aliceVision_main(int argc, char** argv) } image::Image labels; - if (!computeWTALabels(labels, overlaps, warpingFolder, referenceBoundingBox, panoramaMap->getWidth())) + if (!computeWTALabels(labels, overlaps, warpingFolder, *panoramaMap, viewReference)) { ALICEVISION_LOG_ERROR("Error estimating panorama labels for this input"); return false; } - { + /*{ const std::string viewIdStr = std::to_string(viewReference); const std::string outputFilePath = (fs::path(outputFolder) / (viewIdStr + "_label.exr")).string(); image::writeImage(outputFilePath, labels, image::EImageColorSpace::NO_CONVERSION); - } + }*/ int posCurrent = 0; for (IndexT viewCurrent : overlaps) @@ -375,47 +353,53 @@ int aliceVision_main(int argc, char** argv) image::Image mask; image::readImage(maskPath, mask, image::EImageColorSpace::NO_CONVERSION); - // Retrieve position of image in panorama - oiio::ParamValueList metadata = image::readImageMetadata(imagePath); - int offsetX = metadata.find("AliceVision:offsetX")->get_int(); - int offsetY = metadata.find("AliceVision:offsetY")->get_int(); - - if (offsetX > referenceBoundingBox.getRight()) + std::vector intersections; + std::vector currentBoundingBoxes; + if (!panoramaMap->getIntersectionsList(intersections, currentBoundingBoxes, viewReference, viewCurrent)) { - offsetX -= panoramaMap->getWidth(); + continue; } - - image::Image weights; - /*const std::string weightsPath = (fs::path(warpingFolder) / (std::to_string(viewCurrent) + "_weight.exr")).string(); - ALICEVISION_LOG_TRACE("Load weights with path " << weightsPath); - image::readImage(weightsPath, weights, image::EImageColorSpace::NO_CONVERSION);*/ - weights = image::Image(mask.Width(), mask.Height()); - if (!getMaskFromLabels(weights, labels, viewCurrent, offsetX - referenceBoundingBox.left, offsetY - referenceBoundingBox.top)) + for (int indexIntersection = 0; indexIntersection < intersections.size(); indexIntersection++) { - ALICEVISION_LOG_ERROR("Error estimating seams image"); - return EXIT_FAILURE; - } - - if (!compositer->append(source, mask, weights, offsetX - referenceBoundingBox.left, offsetY - referenceBoundingBox.top)) - { - ALICEVISION_LOG_INFO("Error in compositer append"); - return EXIT_FAILURE; - } + const BoundingBox & bbox = currentBoundingBoxes[indexIntersection]; + const BoundingBox & bboxIntersect = intersections[indexIntersection]; + + BoundingBox cutBoundingBox; + cutBoundingBox.left = bboxIntersect.left - bbox.left; + cutBoundingBox.top = bboxIntersect.top - bbox.top; + cutBoundingBox.width = bboxIntersect.width; + cutBoundingBox.height = bboxIntersect.height; + if (cutBoundingBox.isEmpty()) + { + continue; + } - if (mask.Width() >= panoramaMap->getWidth() || referenceBoundingBox.width >= panoramaMap->getWidth()) - { - if (!getMaskFromLabels(weights, labels, viewCurrent, offsetX - panoramaMap->getWidth() - referenceBoundingBox.left, offsetY - referenceBoundingBox.top)) + /*const std::string weightsPath = (fs::path(warpingFolder) / (std::to_string(viewCurrent) + "_weight.exr")).string(); + ALICEVISION_LOG_TRACE("Load weights with path " << weightsPath); + image::readImage(weightsPath, weights, image::EImageColorSpace::NO_CONVERSION);*/ + image::Image weights; + weights = image::Image(mask.Width(), mask.Height()); + + if (!getMaskFromLabels(weights, labels, viewCurrent, bbox.left - referenceBoundingBox.left, bbox.top - referenceBoundingBox.top)) { ALICEVISION_LOG_ERROR("Error estimating seams image"); return EXIT_FAILURE; } - if (!compositer->append(source, mask, weights, offsetX - panoramaMap->getWidth() - referenceBoundingBox.left, offsetY - referenceBoundingBox.top)) + image::Image subsource(cutBoundingBox.width, cutBoundingBox.height); + image::Image submask(cutBoundingBox.width, cutBoundingBox.height); + image::Image subweights(cutBoundingBox.width, cutBoundingBox.height); + + subsource = source.block(cutBoundingBox.top, cutBoundingBox.left, cutBoundingBox.height, cutBoundingBox.width); + submask = mask.block(cutBoundingBox.top, cutBoundingBox.left, cutBoundingBox.height, cutBoundingBox.width); + subweights = weights.block(cutBoundingBox.top, cutBoundingBox.left, cutBoundingBox.height, cutBoundingBox.width); + + if (!compositer->append(subsource, submask, subweights, bboxIntersect.left - referenceBoundingBox.left, bboxIntersect.top - referenceBoundingBox.top)) { ALICEVISION_LOG_INFO("Error in compositer append"); return EXIT_FAILURE; - } + } } } From 708874adb28b2c05fbace35c0b73b3b707e5f68e Mon Sep 17 00:00:00 2001 From: Fabien Castan Date: Wed, 9 Dec 2020 14:23:12 +0100 Subject: [PATCH 50/79] [panorama] minor formatting --- src/aliceVision/image/cache.cpp | 8 ++++---- src/aliceVision/image/cache.hpp | 16 ++++++++-------- src/aliceVision/panorama/cachedImage.hpp | 4 ++-- src/aliceVision/panorama/graphcut.hpp | 6 +++--- src/aliceVision/panorama/imageOps.hpp | 4 ++-- src/aliceVision/panorama/seams.cpp | 2 +- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/src/aliceVision/image/cache.cpp b/src/aliceVision/image/cache.cpp index 8ed7689d54..70e383686e 100644 --- a/src/aliceVision/image/cache.cpp +++ b/src/aliceVision/image/cache.cpp @@ -11,11 +11,11 @@ namespace image { CacheManager::CacheManager(const std::string & pathStorage, size_t blockSize, size_t maxBlocksPerIndex) : -_basePathStorage(pathStorage), _blockSize(blockSize), _incoreBlockUsageCount(0), _incoreBlockUsageMax(10), -_blockCountPerIndex(maxBlocksPerIndex) +_blockCountPerIndex(maxBlocksPerIndex), +_basePathStorage(pathStorage) { wipe(); } @@ -323,8 +323,8 @@ bool CachedTile::acquire() { return true; } -TileCacheManager::TileCacheManager(const std::string & path_storage, size_t tileWidth, size_t tileHeight, size_t maxTilesPerIndex) : -CacheManager(path_storage, tileWidth * tileHeight, maxTilesPerIndex), +TileCacheManager::TileCacheManager(const std::string & pathStorage, size_t tileWidth, size_t tileHeight, size_t maxTilesPerIndex) : +CacheManager(pathStorage, tileWidth * tileHeight, maxTilesPerIndex), _tileWidth(tileWidth), _tileHeight(tileHeight) { } diff --git a/src/aliceVision/image/cache.hpp b/src/aliceVision/image/cache.hpp index a84b0b28a1..4d87f13041 100644 --- a/src/aliceVision/image/cache.hpp +++ b/src/aliceVision/image/cache.hpp @@ -239,13 +239,13 @@ class CacheManager { protected: - size_t _blockSize; - size_t _incoreBlockUsageCount; - size_t _incoreBlockUsageMax; - size_t _blockCountPerIndex; - size_t _nextStartBlockId; - size_t _nextObjectId; - + size_t _blockSize{0}; + size_t _incoreBlockUsageCount{0}; + size_t _incoreBlockUsageMax{0}; + size_t _blockCountPerIndex{0}; + size_t _nextStartBlockId{0}; + size_t _nextObjectId{0}; + std::string _basePathStorage; IndexedStoragePaths _indexPaths; IndexedFreeBlocks _freeBlocks; @@ -323,7 +323,7 @@ class TileCacheManager : public CacheManager, public std::enable_shared_from_thi protected: - TileCacheManager(const std::string & path_storage, size_t tileWidth, size_t tileHeight, size_t maxTilesPerIndex); + TileCacheManager(const std::string & pathStorage, size_t tileWidth, size_t tileHeight, size_t maxTilesPerIndex); virtual void onRemovedFromMRU(size_t objectId); diff --git a/src/aliceVision/panorama/cachedImage.hpp b/src/aliceVision/panorama/cachedImage.hpp index 55219d179c..c7042bb64e 100644 --- a/src/aliceVision/panorama/cachedImage.hpp +++ b/src/aliceVision/panorama/cachedImage.hpp @@ -195,7 +195,7 @@ class CachedImage T * dataSource = (T*)ptrSource->getDataPointer(); std::memcpy(data, dataSource, _tileSize * _tileSize * sizeof(T)); - } + } } return true; @@ -498,4 +498,4 @@ bool CachedImage::writeImage(const std::string& path, const image::EStora template <> bool CachedImage::writeImage(const std::string& path, const image::EStorageDataType &storageDataType); -} // namespace aliceVision \ No newline at end of file +} // namespace aliceVision diff --git a/src/aliceVision/panorama/graphcut.hpp b/src/aliceVision/panorama/graphcut.hpp index 924f0c79ac..1122345ade 100644 --- a/src/aliceVision/panorama/graphcut.hpp +++ b/src/aliceVision/panorama/graphcut.hpp @@ -599,7 +599,7 @@ class GraphcutSeams if (!computeAlphaExpansion) { for (auto & info : _inputs) - { + { double cost; ALICEVISION_LOG_INFO("process input"); @@ -619,7 +619,7 @@ class GraphcutSeams } for (int i = 0; i < 10; i++) - { + { ALICEVISION_LOG_INFO("GraphCut processing iteration #" << i); // For each possible label, try to extends its domination on the label's world bool hasChange = false; @@ -1045,4 +1045,4 @@ class GraphcutSeams CachedImage _labels; }; -} // namespace aliceVision \ No newline at end of file +} // namespace aliceVision diff --git a/src/aliceVision/panorama/imageOps.hpp b/src/aliceVision/panorama/imageOps.hpp index 6f3ae5ee88..90d98c4858 100644 --- a/src/aliceVision/panorama/imageOps.hpp +++ b/src/aliceVision/panorama/imageOps.hpp @@ -320,7 +320,7 @@ bool makeImagePyramidCompatible(image::Image& output, double low_width = large_width * maxScale; double low_height = large_height * maxScale; - /*Make sure width is integer event at the lowest level*/ + /*Make sure width is integer even at the lowest level*/ double correctedLowWidth = ceil(low_width); double correctedLowHeight = ceil(low_height); @@ -339,4 +339,4 @@ bool makeImagePyramidCompatible(image::Image& output, } -} // namespace aliceVision \ No newline at end of file +} // namespace aliceVision diff --git a/src/aliceVision/panorama/seams.cpp b/src/aliceVision/panorama/seams.cpp index 7109bf3334..a011ef0c93 100644 --- a/src/aliceVision/panorama/seams.cpp +++ b/src/aliceVision/panorama/seams.cpp @@ -653,4 +653,4 @@ bool getMaskFromLabels(aliceVision::image::Image & mask, image::Image Date: Thu, 17 Dec 2020 12:40:18 +0100 Subject: [PATCH 51/79] [panorama] memory leak --- src/aliceVision/panorama/compositer.hpp | 6 +++++- src/aliceVision/panorama/laplacianCompositer.hpp | 5 +++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/aliceVision/panorama/compositer.hpp b/src/aliceVision/panorama/compositer.hpp index f22d0184fe..5c8ca6a0d3 100644 --- a/src/aliceVision/panorama/compositer.hpp +++ b/src/aliceVision/panorama/compositer.hpp @@ -19,6 +19,11 @@ class Compositer } + virtual ~Compositer() + { + + } + virtual bool append(const aliceVision::image::Image& color, const aliceVision::image::Image& inputMask, const aliceVision::image::Image& inputWeights, @@ -94,7 +99,6 @@ class Compositer } protected: - image::TileCacheManager::shared_ptr _cacheManager; image::Image _panorama; int _panoramaWidth; int _panoramaHeight; diff --git a/src/aliceVision/panorama/laplacianCompositer.hpp b/src/aliceVision/panorama/laplacianCompositer.hpp index 98ff639ffa..c15333b67f 100644 --- a/src/aliceVision/panorama/laplacianCompositer.hpp +++ b/src/aliceVision/panorama/laplacianCompositer.hpp @@ -17,6 +17,11 @@ class LaplacianCompositer : public Compositer { } + virtual ~LaplacianCompositer() + { + + } + virtual bool initialize() { if (!Compositer::initialize()) From f8ded9472b9db2121db1eb51669d652d09a788ba Mon Sep 17 00:00:00 2001 From: Fabien Date: Thu, 17 Dec 2020 13:25:34 +0100 Subject: [PATCH 52/79] [panorama] optimize chunks distribution --- src/aliceVision/panorama/panoramaMap.cpp | 37 ++++++++++++++++++ src/aliceVision/panorama/panoramaMap.hpp | 1 + .../pipeline/main_panoramaCompositing.cpp | 39 +++++++++++-------- 3 files changed, 60 insertions(+), 17 deletions(-) diff --git a/src/aliceVision/panorama/panoramaMap.cpp b/src/aliceVision/panorama/panoramaMap.cpp index 072e69e9fc..e88e165428 100644 --- a/src/aliceVision/panorama/panoramaMap.cpp +++ b/src/aliceVision/panorama/panoramaMap.cpp @@ -133,4 +133,41 @@ bool PanoramaMap::getIntersectionsList(std::vector & intersections, return true; } +bool PanoramaMap::optimizeChunks(std::vector> & chunks, int chunkSize) { + + int countViews = _map.size(); + int countChunks = int(std::ceil(double(countViews) / double(chunkSize))); + + chunks.clear(); + chunks.resize(countChunks); + + for (auto item : _map) + { + std::sort(chunks.begin(), chunks.end(), + [this](const std::vector & first, const std::vector & second) + { + size_t size_first = 0; + for (int i = 0; i < first.size(); i++) + { + IndexT curIndex = first[i]; + size_first += _map.at(curIndex).area(); + } + + size_t size_second = 0; + for (int i = 0; i < second.size(); i++) + { + IndexT curIndex = second[i]; + size_second += _map.at(curIndex).area(); + } + + return (size_first < size_second); + } + ); + + chunks[0].push_back(item.first); + } + + return true; +} + } \ No newline at end of file diff --git a/src/aliceVision/panorama/panoramaMap.hpp b/src/aliceVision/panorama/panoramaMap.hpp index 6649e9beb0..5e972ae0bb 100644 --- a/src/aliceVision/panorama/panoramaMap.hpp +++ b/src/aliceVision/panorama/panoramaMap.hpp @@ -54,6 +54,7 @@ class PanoramaMap bool getIntersectionsList(std::vector & intersections, std::vector & currentBoundingBoxes, const IndexT & referenceIndex, const IndexT & otherIndex) const; + bool optimizeChunks(std::vector> & chunks, int chunkSize); private: bool intersect(const BoundingBox & box1, const BoundingBox & box2) const; diff --git a/src/software/pipeline/main_panoramaCompositing.cpp b/src/software/pipeline/main_panoramaCompositing.cpp index d11d2d296f..80b96fba82 100644 --- a/src/software/pipeline/main_panoramaCompositing.cpp +++ b/src/software/pipeline/main_panoramaCompositing.cpp @@ -174,7 +174,7 @@ int aliceVision_main(int argc, char** argv) std::string outputFolder; std::string temporaryCachePath; std::string compositerType = "multiband"; - int rangeStart = -1; + int rangeIteration = -1; int rangeSize = 1; image::EStorageDataType storageDataType = image::EStorageDataType::Float; @@ -200,7 +200,7 @@ int aliceVision_main(int argc, char** argv) optionalParams.add_options() ("compositerType,c", po::value(&compositerType)->required(), "Compositer Type [replace, alpha, multiband].") ("storageDataType", po::value(&storageDataType)->default_value(storageDataType), ("Storage data type: " + image::EStorageDataType_informations()).c_str()) - ("rangeStart", po::value(&rangeStart)->default_value(rangeStart), "Range image index start.") + ("rangeIteration", po::value(&rangeIteration)->default_value(rangeIteration), "Range chunk id.") ("rangeSize", po::value(&rangeSize)->default_value(rangeSize), "Range size."); allParams.add(optionalParams); @@ -251,24 +251,26 @@ int aliceVision_main(int argc, char** argv) } // Define range to compute - size_t viewsCount = sfmData.getViews().size(); - if(rangeStart != -1) + int viewsCount = sfmData.getViews().size(); + if(rangeIteration != -1) { - if(rangeStart < 0 || rangeSize < 0 || - std::size_t(rangeStart) > viewsCount) + if(rangeIteration < 0 || rangeSize < 0) { ALICEVISION_LOG_ERROR("Range is incorrect"); return EXIT_FAILURE; } - if(std::size_t(rangeStart + rangeSize) > viewsCount) + int countIterations = int(std::ceil(double(viewsCount) / double(rangeSize))); + + if(rangeIteration >= countIterations) { - rangeSize = int(viewsCount) - rangeStart; + ALICEVISION_LOG_ERROR("Range is incorrect"); + return EXIT_FAILURE; } } else { - rangeStart = 0; + rangeIteration = 0; rangeSize = int(viewsCount); } @@ -282,19 +284,22 @@ int aliceVision_main(int argc, char** argv) return EXIT_FAILURE; } - // Build a list of views - std::vector> views; - for (auto & vIterator : sfmData.getViews()) + // Distribute more smartly inputs among chunks + std::vector> chunks; + if (!panoramaMap->optimizeChunks(chunks, rangeSize)) { - views.push_back(vIterator.second); + ALICEVISION_LOG_ERROR("Can't build chunks"); + return EXIT_FAILURE; } + - for (std::size_t posReference = std::size_t(rangeStart); posReference < std::size_t(rangeStart + rangeSize); ++posReference) + const std::vector & chunk = chunks[rangeIteration]; + for (std::size_t posReference = 0; posReference < chunk.size(); posReference++) { - ALICEVISION_LOG_INFO("processing input region " << posReference + 1 << "/" << views.size()); + ALICEVISION_LOG_INFO("processing input region " << posReference + 1 << "/" << chunk.size()); + + IndexT viewReference = chunk[posReference]; - IndexT viewReference = views[posReference]->getViewId(); - //if (viewReference != 137597937) continue; // Get the list of input which should be processed for this reference view bounding box std::list overlaps; if (!panoramaMap->getOverlaps(overlaps, viewReference)) From 0666c91fde9ae4e1da0a229ed6dd14f9e5556a13 Mon Sep 17 00:00:00 2001 From: Fabien Date: Fri, 18 Dec 2020 08:37:10 +0100 Subject: [PATCH 53/79] [panorama] add merging node for panorama final creation --- src/aliceVision/panorama/compositer.hpp | 30 +--- src/aliceVision/panorama/panoramaMap.hpp | 2 +- src/software/pipeline/CMakeLists.txt | 10 ++ .../pipeline/main_panoramaCompositing.cpp | 41 +++-- .../pipeline/main_panoramaMerging.cpp | 168 ++++++++++++++++++ 5 files changed, 212 insertions(+), 39 deletions(-) create mode 100644 src/software/pipeline/main_panoramaMerging.cpp diff --git a/src/aliceVision/panorama/compositer.hpp b/src/aliceVision/panorama/compositer.hpp index 5c8ca6a0d3..226cccf18a 100644 --- a/src/aliceVision/panorama/compositer.hpp +++ b/src/aliceVision/panorama/compositer.hpp @@ -63,34 +63,10 @@ class Compositer virtual bool terminate() { return true; } - bool save(const std::string &path, const image::EStorageDataType &storageDataType) + + image::Image & getOutput() { - if (storageDataType == image::EStorageDataType::HalfFinite) - { - for (int i = 0; i < _panorama.Height(); i++) - { - for (int j = 0; j < _panorama.Width(); j++) - { - image::RGBAfColor ret; - image::RGBAfColor c = _panorama(i, j); - - const float limit = float(HALF_MAX); - - ret.r() = clamp(c.r(), -limit, limit); - ret.g() = clamp(c.g(), -limit, limit); - ret.b() = clamp(c.b(), -limit, limit); - ret.a() = c.a(); - - _panorama(i, j) = ret; - } - } - } - - oiio::ParamValueList metadata; - metadata.push_back(oiio::ParamValue("AliceVision:storageDataType", EStorageDataType_enumToString(storageDataType))); - image::writeImage(path, _panorama, image::EImageColorSpace::LINEAR, metadata); - - return true; + return _panorama; } virtual int getBorderSize() const diff --git a/src/aliceVision/panorama/panoramaMap.hpp b/src/aliceVision/panorama/panoramaMap.hpp index 5e972ae0bb..a033ad28da 100644 --- a/src/aliceVision/panorama/panoramaMap.hpp +++ b/src/aliceVision/panorama/panoramaMap.hpp @@ -30,7 +30,7 @@ class PanoramaMap return _panoramaWidth; } - size_t geHeight() const + size_t getHeight() const { return _panoramaHeight; } diff --git a/src/software/pipeline/CMakeLists.txt b/src/software/pipeline/CMakeLists.txt index 151d17f4bb..0a8d813943 100644 --- a/src/software/pipeline/CMakeLists.txt +++ b/src/software/pipeline/CMakeLists.txt @@ -137,6 +137,16 @@ if(ALICEVISION_BUILD_SFM) aliceVision_panorama ${Boost_LIBRARIES} ) + alicevision_add_software(aliceVision_panoramaMerging + SOURCE main_panoramaMerging.cpp + FOLDER ${FOLDER_SOFTWARE_PIPELINE} + LINKS aliceVision_system + aliceVision_image + aliceVision_sfmData + aliceVision_sfmDataIO + aliceVision_panorama + ${Boost_LIBRARIES} + ) alicevision_add_software(aliceVision_panoramaCompositing SOURCE main_panoramaCompositing.cpp FOLDER ${FOLDER_SOFTWARE_PIPELINE} diff --git a/src/software/pipeline/main_panoramaCompositing.cpp b/src/software/pipeline/main_panoramaCompositing.cpp index 80b96fba82..3bf9654ec8 100644 --- a/src/software/pipeline/main_panoramaCompositing.cpp +++ b/src/software/pipeline/main_panoramaCompositing.cpp @@ -172,7 +172,6 @@ int aliceVision_main(int argc, char** argv) std::string sfmDataFilepath; std::string warpingFolder; std::string outputFolder; - std::string temporaryCachePath; std::string compositerType = "multiband"; int rangeIteration = -1; int rangeSize = 1; @@ -191,8 +190,7 @@ int aliceVision_main(int argc, char** argv) requiredParams.add_options() ("input,i", po::value(&sfmDataFilepath)->required(), "Input sfmData.") ("warpingFolder,w", po::value(&warpingFolder)->required(), "Folder with warped images.") - ("output,o", po::value(&outputFolder)->required(), "Path of the output panorama.") - ("cacheFolder,f", po::value(&temporaryCachePath)->required(), "Path of the temporary cache."); + ("output,o", po::value(&outputFolder)->required(), "Path of the output panorama."); allParams.add(requiredParams); // Description of optional parameters @@ -333,12 +331,6 @@ int aliceVision_main(int argc, char** argv) ALICEVISION_LOG_ERROR("Error estimating panorama labels for this input"); return false; } - - /*{ - const std::string viewIdStr = std::to_string(viewReference); - const std::string outputFilePath = (fs::path(outputFolder) / (viewIdStr + "_label.exr")).string(); - image::writeImage(outputFilePath, labels, image::EImageColorSpace::NO_CONVERSION); - }*/ int posCurrent = 0; for (IndexT viewCurrent : overlaps) @@ -417,10 +409,37 @@ int aliceVision_main(int argc, char** argv) const std::string viewIdStr = std::to_string(viewReference); const std::string outputFilePath = (fs::path(outputFolder) / (viewIdStr + ".exr")).string(); - if (!compositer->save(outputFilePath, storageDataType)) + image::Image output = compositer->getOutput(); + + if (storageDataType == image::EStorageDataType::HalfFinite) { - ALICEVISION_LOG_ERROR("Error terminating panorama"); + for (int i = 0; i < output.Height(); i++) + { + for (int j = 0; j < output.Width(); j++) + { + image::RGBAfColor ret; + image::RGBAfColor c = output(i, j); + + const float limit = float(HALF_MAX); + + ret.r() = clamp(c.r(), -limit, limit); + ret.g() = clamp(c.g(), -limit, limit); + ret.b() = clamp(c.b(), -limit, limit); + ret.a() = c.a(); + + output(i, j) = ret; + } + } } + + oiio::ParamValueList metadata; + metadata.push_back(oiio::ParamValue("AliceVision:storageDataType", EStorageDataType_enumToString(storageDataType))); + metadata.push_back(oiio::ParamValue("AliceVision:offsetX", int(referenceBoundingBox.left))); + metadata.push_back(oiio::ParamValue("AliceVision:offsetY", int(referenceBoundingBox.top))); + metadata.push_back(oiio::ParamValue("AliceVision:panoramaWidth", int(panoramaMap->getWidth()))); + metadata.push_back(oiio::ParamValue("AliceVision:panoramaHeight", int(panoramaMap->getHeight()))); + + image::writeImage(outputFilePath, output, image::EImageColorSpace::LINEAR, metadata); } return EXIT_SUCCESS; diff --git a/src/software/pipeline/main_panoramaMerging.cpp b/src/software/pipeline/main_panoramaMerging.cpp new file mode 100644 index 0000000000..270f6bab0f --- /dev/null +++ b/src/software/pipeline/main_panoramaMerging.cpp @@ -0,0 +1,168 @@ +// This file is part of the AliceVision project. +// Copyright (c) 2020 AliceVision contributors. +// This Source Code Form is subject to the terms of the Mozilla Public License, +// v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at https://mozilla.org/MPL/2.0/. + +// Input and geometry +#include +#include + +// Image +#include +#include + +// System +#include +#include + +// Reading command line options +#include +#include +#include + +// IO +#include +#include +#include +#include +#include + +// These constants define the current software version. +// They must be updated when the command line is changed. +#define ALICEVISION_SOFTWARE_VERSION_MAJOR 1 +#define ALICEVISION_SOFTWARE_VERSION_MINOR 0 + +using namespace aliceVision; + +namespace po = boost::program_options; +namespace bpt = boost::property_tree; +namespace fs = boost::filesystem; + +int aliceVision_main(int argc, char** argv) +{ + std::string sfmDataFilepath; + std::string compositingFolder; + std::string outputPanoramaPath; + image::EStorageDataType storageDataType = image::EStorageDataType::Float; + + system::EVerboseLevel verboseLevel = system::Logger::getDefaultVerboseLevel(); + + // Program description + po::options_description allParams( + "Perform panorama stiching of cameras around a nodal point for 360° panorama creation. \n" + "AliceVision PanoramaCompositing"); + + // Description of mandatory parameters + po::options_description requiredParams("Required parameters"); + requiredParams.add_options() + ("input,i", po::value(&sfmDataFilepath)->required(), "Input sfmData.") + ("compositingFolder,w", po::value(&compositingFolder)->required(), "Folder with composited images.") + ("outputPanorama,o", po::value(&outputPanoramaPath)->required(), "Path of the output panorama."); + allParams.add(requiredParams); + + // Description of optional parameters + po::options_description optionalParams("Optional parameters"); + optionalParams.add_options() + ("storageDataType", po::value(&storageDataType)->default_value(storageDataType), ("Storage data type: " + image::EStorageDataType_informations()).c_str()); + allParams.add(optionalParams); + + // Setup log level given command line + po::options_description logParams("Log parameters"); + logParams.add_options() + ("verboseLevel,v", po::value(&verboseLevel)->default_value(verboseLevel), "verbosity level (fatal, error, warning, info, debug, trace)."); + allParams.add(logParams); + + // Effectively parse command line given parse options + po::variables_map vm; + try + { + po::store(po::parse_command_line(argc, argv, allParams), vm); + + if(vm.count("help") || (argc == 1)) + { + ALICEVISION_COUT(allParams); + return EXIT_SUCCESS; + } + po::notify(vm); + } + catch(boost::program_options::required_option& e) + { + ALICEVISION_CERR("ERROR: " << e.what()); + ALICEVISION_COUT("Usage:\n\n" << allParams); + return EXIT_FAILURE; + } + catch(boost::program_options::error& e) + { + ALICEVISION_CERR("ERROR: " << e.what()); + ALICEVISION_COUT("Usage:\n\n" << allParams); + return EXIT_FAILURE; + } + + ALICEVISION_COUT("Program called with the following parameters:"); + ALICEVISION_COUT(vm); + + // Set verbose level given command line + system::Logger::get()->setLogLevel(verboseLevel); + + // load input scene + sfmData::SfMData sfmData; + if(!sfmDataIO::Load(sfmData, sfmDataFilepath, sfmDataIO::ESfMData(sfmDataIO::VIEWS | sfmDataIO::EXTRINSICS | sfmDataIO::INTRINSICS))) + { + ALICEVISION_LOG_ERROR("The input file '" + sfmDataFilepath + "' cannot be read"); + return EXIT_FAILURE; + } + + + bool first = true; + image::Image panorama; + + for (auto viewItem : sfmData.getViews()) + { + IndexT viewId = viewItem.first; + + // Get composited image path + const std::string imagePath = (fs::path(compositingFolder) / (std::to_string(viewId) + ".exr")).string(); + + // Get offset + oiio::ParamValueList metadata = image::readImageMetadata(imagePath); + const int offsetX = metadata.find("AliceVision:offsetX")->get_int(); + const int offsetY = metadata.find("AliceVision:offsetY")->get_int(); + const int panoramaWidth = metadata.find("AliceVision:panoramaWidth")->get_int(); + const int panoramaHeight = metadata.find("AliceVision:panoramaHeight")->get_int(); + + if (first) + { + panorama = image::Image(panoramaWidth, panoramaHeight, true, image::RGBAfColor(0.0f, 0.0f, 0.f, 0.0f)); + first = false; + } + + // Load image + ALICEVISION_LOG_TRACE("Load image with path " << imagePath); + image::Image source; + image::readImage(imagePath, source, image::EImageColorSpace::NO_CONVERSION); + + for (int i = 0; i < source.Height(); i++) + { + for (int j = 0; j < source.Width(); j++) + { + image::RGBAfColor pix = source(i, j); + + if (pix.a() > 0.9) { + + int nx = offsetX + j; + if (nx < 0) nx += panoramaWidth; + if (nx >= panoramaWidth) nx -= panoramaWidth; + + panorama(offsetY + i, nx) = pix; + } + } + } + } + + oiio::ParamValueList targetMetadata; + targetMetadata.push_back(oiio::ParamValue("AliceVision:storageDataType", image::EStorageDataType_enumToString(storageDataType))); + image::writeImage(outputPanoramaPath, panorama, image::EImageColorSpace::AUTO, targetMetadata); + + return EXIT_SUCCESS; +} From 716a4ee12e6f622d6e099a9bee7320289d6c10af Mon Sep 17 00:00:00 2001 From: Fabien Date: Fri, 18 Dec 2020 09:53:32 +0100 Subject: [PATCH 54/79] [panorama] some memory optimizations --- src/aliceVision/image/io.cpp | 41 +++++++--- src/aliceVision/image/io.hpp | 8 +- src/aliceVision/panorama/alphaCompositer.hpp | 6 +- src/aliceVision/panorama/compositer.hpp | 6 +- .../panorama/laplacianCompositer.hpp | 19 +++-- .../pipeline/main_panoramaCompositing.cpp | 80 +++++++++++++++---- 6 files changed, 120 insertions(+), 40 deletions(-) diff --git a/src/aliceVision/image/io.cpp b/src/aliceVision/image/io.cpp index 12d95201f5..7de21f739c 100644 --- a/src/aliceVision/image/io.cpp +++ b/src/aliceVision/image/io.cpp @@ -269,18 +269,41 @@ void readImage(const std::string& path, oiio::ImageBuf inBuf(path, 0, 0, NULL, &configSpec); - inBuf.read(0, 0, true, oiio::TypeDesc::FLOAT); // force image convertion to float (for grayscale and color space convertion) + //Handle ROI option + oiio::ROI customROI = inBuf.roi(); + oiio::ROI subROI = imageReadOptions.subROI; + if (imageReadOptions.subROI.defined()) + { + //Make sure both roi are in the same frame + subROI.xbegin += inBuf.roi().xbegin; + subROI.ybegin += inBuf.roi().ybegin; + + //Keep the intersection + customROI = oiio::roi_intersection(subROI, customROI); + customROI.zbegin = inBuf.roi().zbegin; + customROI.zend = inBuf.roi().zend; + customROI.chbegin = inBuf.roi().chbegin; + customROI.chend = inBuf.roi().chend; + } + + inBuf.read(0, 0, false, oiio::TypeDesc::FLOAT); // force image convertion to float (for grayscale and color space convertion) if(!inBuf.initialized()) + { throw std::runtime_error("Cannot find/open image file '" + path + "'."); + } // check picture channels number if(inBuf.spec().nchannels != 1 && inBuf.spec().nchannels < 3) + { throw std::runtime_error("Can't load channels of image file '" + path + "'."); + } // color conversion if(imageReadOptions.outputColorSpace == EImageColorSpace::AUTO) + { throw std::runtime_error("You must specify a requested color space for image file '" + path + "'."); + } const std::string& colorSpace = inBuf.spec().get_string_attribute("oiio:ColorSpace", "sRGB"); // default image color space is sRGB ALICEVISION_LOG_TRACE("Read image " << path << " (encoded in " << colorSpace << " colorspace)."); @@ -289,7 +312,7 @@ void readImage(const std::string& path, { if (colorSpace != "sRGB") { - oiio::ImageBufAlgo::colorconvert(inBuf, inBuf, colorSpace, "sRGB"); + oiio::ImageBufAlgo::colorconvert(inBuf, inBuf, colorSpace, "sRGB", true, "", "", nullptr, customROI); ALICEVISION_LOG_TRACE("Convert image " << path << " from " << colorSpace << " to sRGB colorspace"); } } @@ -297,7 +320,7 @@ void readImage(const std::string& path, { if (colorSpace != "Linear") { - oiio::ImageBufAlgo::colorconvert(inBuf, inBuf, colorSpace, "Linear"); + oiio::ImageBufAlgo::colorconvert(inBuf, inBuf, colorSpace, "Linear", true, "", "", nullptr, customROI); ALICEVISION_LOG_TRACE("Convert image " << path << " from " << colorSpace << " to Linear colorspace"); } } @@ -306,15 +329,15 @@ void readImage(const std::string& path, if(nchannels == 1 && inBuf.spec().nchannels >= 3) { // convertion region of interest (for inBuf.spec().nchannels > 3) - oiio::ROI convertionROI = inBuf.roi(); - convertionROI.chbegin = 0; - convertionROI.chend = 3; + oiio::ROI convertionROI = customROI; + customROI.chbegin = 0; + customROI.chend = 3; // compute luminance via a weighted sum of R,G,B // (assuming Rec709 primaries and a linear scale) const float weights[3] = {.2126f, .7152f, .0722f}; oiio::ImageBuf grayscaleBuf; - oiio::ImageBufAlgo::channel_sum(grayscaleBuf, inBuf, weights, convertionROI); + oiio::ImageBufAlgo::channel_sum(grayscaleBuf, inBuf, weights, customROI); inBuf.copy(grayscaleBuf); // TODO: if inBuf.spec().nchannels == 4: premult? @@ -346,9 +369,9 @@ void readImage(const std::string& path, } // copy pixels from oiio to eigen - image.resize(inBuf.spec().width, inBuf.spec().height, false); + image.resize(customROI.width(), customROI.height(), false); { - oiio::ROI exportROI = inBuf.roi(); + oiio::ROI exportROI = customROI; exportROI.chbegin = 0; exportROI.chend = nchannels; diff --git a/src/aliceVision/image/io.hpp b/src/aliceVision/image/io.hpp index d92e447c21..a0bbc5800a 100644 --- a/src/aliceVision/image/io.hpp +++ b/src/aliceVision/image/io.hpp @@ -48,13 +48,17 @@ enum class EImageFileType */ struct ImageReadOptions { - ImageReadOptions(EImageColorSpace colorSpace = EImageColorSpace::AUTO, bool useWhiteBalance = true) : - outputColorSpace(colorSpace), applyWhiteBalance(useWhiteBalance) + ImageReadOptions(EImageColorSpace colorSpace = EImageColorSpace::AUTO, bool useWhiteBalance = true, const oiio::ROI & roi = oiio::ROI()) : + outputColorSpace(colorSpace), applyWhiteBalance(useWhiteBalance), subROI(roi) { } EImageColorSpace outputColorSpace; bool applyWhiteBalance; + + //ROI for this image. + //If the image contains an roi, this is the roi INSIDE the roi. + oiio::ROI subROI; }; diff --git a/src/aliceVision/panorama/alphaCompositer.hpp b/src/aliceVision/panorama/alphaCompositer.hpp index 7a17b9fde1..460268e601 100644 --- a/src/aliceVision/panorama/alphaCompositer.hpp +++ b/src/aliceVision/panorama/alphaCompositer.hpp @@ -13,9 +13,9 @@ class AlphaCompositer : public Compositer { } - virtual bool append(const aliceVision::image::Image& color, - const aliceVision::image::Image& inputMask, - const aliceVision::image::Image& inputWeights, + virtual bool append(aliceVision::image::Image& color, + aliceVision::image::Image& inputMask, + aliceVision::image::Image& inputWeights, int offset_x, int offset_y) { for(int i = 0; i < color.Height(); i++) diff --git a/src/aliceVision/panorama/compositer.hpp b/src/aliceVision/panorama/compositer.hpp index 226cccf18a..4a63aed29d 100644 --- a/src/aliceVision/panorama/compositer.hpp +++ b/src/aliceVision/panorama/compositer.hpp @@ -24,9 +24,9 @@ class Compositer } - virtual bool append(const aliceVision::image::Image& color, - const aliceVision::image::Image& inputMask, - const aliceVision::image::Image& inputWeights, + virtual bool append(aliceVision::image::Image& color, + aliceVision::image::Image& inputMask, + aliceVision::image::Image& inputWeights, int offset_x, int offset_y) { for(int i = 0; i < color.Height(); i++) diff --git a/src/aliceVision/panorama/laplacianCompositer.hpp b/src/aliceVision/panorama/laplacianCompositer.hpp index c15333b67f..b819f99dcc 100644 --- a/src/aliceVision/panorama/laplacianCompositer.hpp +++ b/src/aliceVision/panorama/laplacianCompositer.hpp @@ -24,11 +24,6 @@ class LaplacianCompositer : public Compositer virtual bool initialize() { - if (!Compositer::initialize()) - { - return false; - } - return _pyramidPanorama.initialize(); } @@ -37,9 +32,9 @@ class LaplacianCompositer : public Compositer return _gaussianFilterRadius; } - virtual bool append(const aliceVision::image::Image& color, - const aliceVision::image::Image& inputMask, - const aliceVision::image::Image& inputWeights, + virtual bool append(aliceVision::image::Image& color, + aliceVision::image::Image& inputMask, + aliceVision::image::Image& inputWeights, int offsetX, int offsetY) { // Make sure input is compatible with pyramid processing @@ -51,10 +46,16 @@ class LaplacianCompositer : public Compositer makeImagePyramidCompatible(colorPot, newOffsetX, newOffsetY, color, offsetX, offsetY, getBorderSize(), _bands); + color = aliceVision::image::Image(); + makeImagePyramidCompatible(maskPot, newOffsetX, newOffsetY, inputMask, offsetX, offsetY, getBorderSize(), _bands); + inputMask = aliceVision::image::Image(); + makeImagePyramidCompatible(weightsPot, newOffsetX, newOffsetY, inputWeights, offsetX, offsetY, getBorderSize(), _bands); + inputWeights = aliceVision::image::Image(); + // Fill Color images masked parts with fake but coherent info aliceVision::image::Image feathered; if (!feathering(feathered, colorPot, maskPot)) @@ -100,6 +101,8 @@ class LaplacianCompositer : public Compositer virtual bool terminate() { + _panorama = image::Image(_panoramaWidth, _panoramaHeight, true, image::RGBAfColor(0.0f, 0.0f, 0.0f, 0.0f)); + if (!_pyramidPanorama.rebuild(_panorama)) { return false; diff --git a/src/software/pipeline/main_panoramaCompositing.cpp b/src/software/pipeline/main_panoramaCompositing.cpp index 3bf9654ec8..506d2dd54f 100644 --- a/src/software/pipeline/main_panoramaCompositing.cpp +++ b/src/software/pipeline/main_panoramaCompositing.cpp @@ -272,9 +272,22 @@ int aliceVision_main(int argc, char** argv) rangeSize = int(viewsCount); } + size_t borderSize; + if (compositerType == "multiband") + { + borderSize = 2; + } + else if (compositerType == "alpha") + { + borderSize = 0; + } + else + { + borderSize = 0; + } + // Build the map of inputs in the final panorama // This is mostly meant to compute overlaps between inputs - const size_t borderSize = 2; std::unique_ptr panoramaMap = buildMap(sfmData, warpingFolder, borderSize); if (viewsCount == 0) { @@ -317,19 +330,45 @@ int aliceVision_main(int argc, char** argv) return EXIT_FAILURE; } + //Create a compositer depending on what we need + bool needWeights; + bool needSeams; std::unique_ptr compositer; - compositer = std::unique_ptr(new LaplacianCompositer(referenceBoundingBox.width, referenceBoundingBox.height, panoramaMap->getScale())); + if (compositerType == "multiband") + { + needWeights = false; + needSeams = true; + compositer = std::unique_ptr(new LaplacianCompositer(referenceBoundingBox.width, referenceBoundingBox.height, panoramaMap->getScale())); + } + else if (compositerType == "alpha") + { + needWeights = true; + needSeams = false; + compositer = std::unique_ptr(new AlphaCompositer(referenceBoundingBox.width, referenceBoundingBox.height)); + } + else + { + needWeights = false; + needSeams = false; + compositer = std::unique_ptr(new Compositer(referenceBoundingBox.width, referenceBoundingBox.height)); + } + + // Compositer initialization if (!compositer->initialize()) { ALICEVISION_LOG_ERROR("Error initializing panorama"); return false; } + // Compute initial seams image::Image labels; - if (!computeWTALabels(labels, overlaps, warpingFolder, *panoramaMap, viewReference)) + if (needSeams) { - ALICEVISION_LOG_ERROR("Error estimating panorama labels for this input"); - return false; + if (!computeWTALabels(labels, overlaps, warpingFolder, *panoramaMap, viewReference)) + { + ALICEVISION_LOG_ERROR("Error estimating panorama labels for this input"); + return false; + } } int posCurrent = 0; @@ -350,6 +389,7 @@ int aliceVision_main(int argc, char** argv) image::Image mask; image::readImage(maskPath, mask, image::EImageColorSpace::NO_CONVERSION); + //Compute list of intersection between this view and the reference view std::vector intersections; std::vector currentBoundingBoxes; if (!panoramaMap->getIntersectionsList(intersections, currentBoundingBoxes, viewReference, viewCurrent)) @@ -372,16 +412,22 @@ int aliceVision_main(int argc, char** argv) continue; } - /*const std::string weightsPath = (fs::path(warpingFolder) / (std::to_string(viewCurrent) + "_weight.exr")).string(); - ALICEVISION_LOG_TRACE("Load weights with path " << weightsPath); - image::readImage(weightsPath, weights, image::EImageColorSpace::NO_CONVERSION);*/ + image::Image weights; - weights = image::Image(mask.Width(), mask.Height()); - - if (!getMaskFromLabels(weights, labels, viewCurrent, bbox.left - referenceBoundingBox.left, bbox.top - referenceBoundingBox.top)) + if (needWeights) { - ALICEVISION_LOG_ERROR("Error estimating seams image"); - return EXIT_FAILURE; + const std::string weightsPath = (fs::path(warpingFolder) / (std::to_string(viewCurrent) + "_weight.exr")).string(); + ALICEVISION_LOG_TRACE("Load weights with path " << weightsPath); + image::readImage(weightsPath, weights, image::EImageColorSpace::NO_CONVERSION); + } + else if (needSeams) + { + weights = image::Image(mask.Width(), mask.Height()); + if (!getMaskFromLabels(weights, labels, viewCurrent, bbox.left - referenceBoundingBox.left, bbox.top - referenceBoundingBox.top)) + { + ALICEVISION_LOG_ERROR("Error estimating seams image"); + return EXIT_FAILURE; + } } image::Image subsource(cutBoundingBox.width, cutBoundingBox.height); @@ -390,7 +436,11 @@ int aliceVision_main(int argc, char** argv) subsource = source.block(cutBoundingBox.top, cutBoundingBox.left, cutBoundingBox.height, cutBoundingBox.width); submask = mask.block(cutBoundingBox.top, cutBoundingBox.left, cutBoundingBox.height, cutBoundingBox.width); - subweights = weights.block(cutBoundingBox.top, cutBoundingBox.left, cutBoundingBox.height, cutBoundingBox.width); + + if (weights.Width() > 0) + { + subweights = weights.block(cutBoundingBox.top, cutBoundingBox.left, cutBoundingBox.height, cutBoundingBox.width); + } if (!compositer->append(subsource, submask, subweights, bboxIntersect.left - referenceBoundingBox.left, bboxIntersect.top - referenceBoundingBox.top)) { @@ -401,7 +451,7 @@ int aliceVision_main(int argc, char** argv) } - std::cout << "terminate" << std::endl; + ALICEVISION_LOG_INFO("Terminate compositing for this view"); if (!compositer->terminate()) { ALICEVISION_LOG_ERROR("Error terminating panorama"); From 5c54183636eafed85f2f09f1fe4f18f34f189dae Mon Sep 17 00:00:00 2001 From: Fabien Date: Fri, 18 Dec 2020 11:19:03 +0100 Subject: [PATCH 55/79] [panorama] enable multithreading for compositing --- .../pipeline/main_panoramaCompositing.cpp | 378 ++++++++++-------- 1 file changed, 201 insertions(+), 177 deletions(-) diff --git a/src/software/pipeline/main_panoramaCompositing.cpp b/src/software/pipeline/main_panoramaCompositing.cpp index 506d2dd54f..a4f8a552ff 100644 --- a/src/software/pipeline/main_panoramaCompositing.cpp +++ b/src/software/pipeline/main_panoramaCompositing.cpp @@ -167,6 +167,191 @@ bool computeWTALabels(image::Image & labels, std::list & views, return true; } +bool processImage(const PanoramaMap & panoramaMap, const std::string & compositerType, const std::string & warpingFolder, const std::string & outputFolder, const image::EStorageDataType & storageDataType, const IndexT & viewReference) +{ + // Get the list of input which should be processed for this reference view bounding box + std::list overlaps; + if (!panoramaMap.getOverlaps(overlaps, viewReference)) + { + ALICEVISION_LOG_ERROR("Problem analyzing neighboorhood"); + return EXIT_FAILURE; + } + + // Add the current input also for simpler processing + overlaps.push_back(viewReference); + + // Get the input bounding box to define the ROI + BoundingBox referenceBoundingBox; + if (!panoramaMap.getBoundingBox(referenceBoundingBox, viewReference)) + { + ALICEVISION_LOG_ERROR("Problem getting reference bounding box"); + return EXIT_FAILURE; + } + + //Create a compositer depending on what we need + bool needWeights; + bool needSeams; + std::unique_ptr compositer; + if (compositerType == "multiband") + { + needWeights = false; + needSeams = true; + compositer = std::unique_ptr(new LaplacianCompositer(referenceBoundingBox.width, referenceBoundingBox.height, panoramaMap.getScale())); + } + else if (compositerType == "alpha") + { + needWeights = true; + needSeams = false; + compositer = std::unique_ptr(new AlphaCompositer(referenceBoundingBox.width, referenceBoundingBox.height)); + } + else + { + needWeights = false; + needSeams = false; + compositer = std::unique_ptr(new Compositer(referenceBoundingBox.width, referenceBoundingBox.height)); + } + + // Compositer initialization + if (!compositer->initialize()) + { + ALICEVISION_LOG_ERROR("Error initializing panorama"); + return false; + } + + // Compute initial seams + image::Image labels; + if (needSeams) + { + if (!computeWTALabels(labels, overlaps, warpingFolder, panoramaMap, viewReference)) + { + ALICEVISION_LOG_ERROR("Error estimating panorama labels for this input"); + return false; + } + } + + int posCurrent = 0; + for (IndexT viewCurrent : overlaps) + { + posCurrent++; + ALICEVISION_LOG_INFO("Processing input " << posCurrent << "/" << overlaps.size()); + + // Load image and convert it to linear colorspace + const std::string imagePath = (fs::path(warpingFolder) / (std::to_string(viewCurrent) + ".exr")).string(); + ALICEVISION_LOG_TRACE("Load image with path " << imagePath); + image::Image source; + image::readImage(imagePath, source, image::EImageColorSpace::NO_CONVERSION); + + // Load mask + const std::string maskPath = (fs::path(warpingFolder) / (std::to_string(viewCurrent) + "_mask.exr")).string(); + ALICEVISION_LOG_TRACE("Load mask with path " << maskPath); + image::Image mask; + image::readImage(maskPath, mask, image::EImageColorSpace::NO_CONVERSION); + + //Compute list of intersection between this view and the reference view + std::vector intersections; + std::vector currentBoundingBoxes; + if (!panoramaMap.getIntersectionsList(intersections, currentBoundingBoxes, viewReference, viewCurrent)) + { + continue; + } + + for (int indexIntersection = 0; indexIntersection < intersections.size(); indexIntersection++) + { + const BoundingBox & bbox = currentBoundingBoxes[indexIntersection]; + const BoundingBox & bboxIntersect = intersections[indexIntersection]; + + BoundingBox cutBoundingBox; + cutBoundingBox.left = bboxIntersect.left - bbox.left; + cutBoundingBox.top = bboxIntersect.top - bbox.top; + cutBoundingBox.width = bboxIntersect.width; + cutBoundingBox.height = bboxIntersect.height; + if (cutBoundingBox.isEmpty()) + { + continue; + } + + + image::Image weights; + if (needWeights) + { + const std::string weightsPath = (fs::path(warpingFolder) / (std::to_string(viewCurrent) + "_weight.exr")).string(); + ALICEVISION_LOG_TRACE("Load weights with path " << weightsPath); + image::readImage(weightsPath, weights, image::EImageColorSpace::NO_CONVERSION); + } + else if (needSeams) + { + weights = image::Image(mask.Width(), mask.Height()); + if (!getMaskFromLabels(weights, labels, viewCurrent, bbox.left - referenceBoundingBox.left, bbox.top - referenceBoundingBox.top)) + { + ALICEVISION_LOG_ERROR("Error estimating seams image"); + return EXIT_FAILURE; + } + } + + image::Image subsource(cutBoundingBox.width, cutBoundingBox.height); + image::Image submask(cutBoundingBox.width, cutBoundingBox.height); + image::Image subweights(cutBoundingBox.width, cutBoundingBox.height); + + subsource = source.block(cutBoundingBox.top, cutBoundingBox.left, cutBoundingBox.height, cutBoundingBox.width); + submask = mask.block(cutBoundingBox.top, cutBoundingBox.left, cutBoundingBox.height, cutBoundingBox.width); + + if (weights.Width() > 0) + { + subweights = weights.block(cutBoundingBox.top, cutBoundingBox.left, cutBoundingBox.height, cutBoundingBox.width); + } + + if (!compositer->append(subsource, submask, subweights, bboxIntersect.left - referenceBoundingBox.left, bboxIntersect.top - referenceBoundingBox.top)) + { + ALICEVISION_LOG_INFO("Error in compositer append"); + return EXIT_FAILURE; + } + } + } + + + ALICEVISION_LOG_INFO("Terminate compositing for this view"); + if (!compositer->terminate()) + { + ALICEVISION_LOG_ERROR("Error terminating panorama"); + } + + const std::string viewIdStr = std::to_string(viewReference); + const std::string outputFilePath = (fs::path(outputFolder) / (viewIdStr + ".exr")).string(); + image::Image output = compositer->getOutput(); + + if (storageDataType == image::EStorageDataType::HalfFinite) + { + for (int i = 0; i < output.Height(); i++) + { + for (int j = 0; j < output.Width(); j++) + { + image::RGBAfColor ret; + image::RGBAfColor c = output(i, j); + + const float limit = float(HALF_MAX); + + ret.r() = clamp(c.r(), -limit, limit); + ret.g() = clamp(c.g(), -limit, limit); + ret.b() = clamp(c.b(), -limit, limit); + ret.a() = c.a(); + + output(i, j) = ret; + } + } + } + + oiio::ParamValueList metadata; + metadata.push_back(oiio::ParamValue("AliceVision:storageDataType", EStorageDataType_enumToString(storageDataType))); + metadata.push_back(oiio::ParamValue("AliceVision:offsetX", int(referenceBoundingBox.left))); + metadata.push_back(oiio::ParamValue("AliceVision:offsetY", int(referenceBoundingBox.top))); + metadata.push_back(oiio::ParamValue("AliceVision:panoramaWidth", int(panoramaMap.getWidth()))); + metadata.push_back(oiio::ParamValue("AliceVision:panoramaHeight", int(panoramaMap.getHeight()))); + + image::writeImage(outputFilePath, output, image::EImageColorSpace::LINEAR, metadata); + + return EXIT_SUCCESS; +} + int aliceVision_main(int argc, char** argv) { std::string sfmDataFilepath; @@ -175,6 +360,7 @@ int aliceVision_main(int argc, char** argv) std::string compositerType = "multiband"; int rangeIteration = -1; int rangeSize = 1; + int maxThreads = 1; image::EStorageDataType storageDataType = image::EStorageDataType::Float; @@ -199,7 +385,8 @@ int aliceVision_main(int argc, char** argv) ("compositerType,c", po::value(&compositerType)->required(), "Compositer Type [replace, alpha, multiband].") ("storageDataType", po::value(&storageDataType)->default_value(storageDataType), ("Storage data type: " + image::EStorageDataType_informations()).c_str()) ("rangeIteration", po::value(&rangeIteration)->default_value(rangeIteration), "Range chunk id.") - ("rangeSize", po::value(&rangeSize)->default_value(rangeSize), "Range size."); + ("rangeSize", po::value(&rangeSize)->default_value(rangeSize), "Range size.") + ("maxThreads", po::value(&maxThreads)->default_value(maxThreads), "max number of threads to use."); allParams.add(optionalParams); // Setup log level given command line @@ -305,191 +492,28 @@ int aliceVision_main(int argc, char** argv) const std::vector & chunk = chunks[rangeIteration]; + + bool succeeded = true; + + omp_set_num_threads(std::min(omp_get_thread_limit(), std::max(0, maxThreads))); + + #pragma omp parallel for for (std::size_t posReference = 0; posReference < chunk.size(); posReference++) { ALICEVISION_LOG_INFO("processing input region " << posReference + 1 << "/" << chunk.size()); IndexT viewReference = chunk[posReference]; - // Get the list of input which should be processed for this reference view bounding box - std::list overlaps; - if (!panoramaMap->getOverlaps(overlaps, viewReference)) - { - ALICEVISION_LOG_ERROR("Problem analyzing neighboorhood"); - return EXIT_FAILURE; - } - - // Add the current input also for simpler processing - overlaps.push_back(viewReference); - - // Get the input bounding box to define the ROI - BoundingBox referenceBoundingBox; - if (!panoramaMap->getBoundingBox(referenceBoundingBox, viewReference)) - { - ALICEVISION_LOG_ERROR("Problem getting reference bounding box"); - return EXIT_FAILURE; - } - - //Create a compositer depending on what we need - bool needWeights; - bool needSeams; - std::unique_ptr compositer; - if (compositerType == "multiband") - { - needWeights = false; - needSeams = true; - compositer = std::unique_ptr(new LaplacianCompositer(referenceBoundingBox.width, referenceBoundingBox.height, panoramaMap->getScale())); - } - else if (compositerType == "alpha") - { - needWeights = true; - needSeams = false; - compositer = std::unique_ptr(new AlphaCompositer(referenceBoundingBox.width, referenceBoundingBox.height)); - } - else - { - needWeights = false; - needSeams = false; - compositer = std::unique_ptr(new Compositer(referenceBoundingBox.width, referenceBoundingBox.height)); - } - - // Compositer initialization - if (!compositer->initialize()) - { - ALICEVISION_LOG_ERROR("Error initializing panorama"); - return false; - } - - // Compute initial seams - image::Image labels; - if (needSeams) - { - if (!computeWTALabels(labels, overlaps, warpingFolder, *panoramaMap, viewReference)) - { - ALICEVISION_LOG_ERROR("Error estimating panorama labels for this input"); - return false; - } - } - - int posCurrent = 0; - for (IndexT viewCurrent : overlaps) + if (!processImage(*panoramaMap, compositerType, warpingFolder, outputFolder, storageDataType, viewReference)) { - posCurrent++; - ALICEVISION_LOG_INFO("Processing input " << posCurrent << "/" << overlaps.size()); - - // Load image and convert it to linear colorspace - const std::string imagePath = (fs::path(warpingFolder) / (std::to_string(viewCurrent) + ".exr")).string(); - ALICEVISION_LOG_TRACE("Load image with path " << imagePath); - image::Image source; - image::readImage(imagePath, source, image::EImageColorSpace::NO_CONVERSION); - - // Load mask - const std::string maskPath = (fs::path(warpingFolder) / (std::to_string(viewCurrent) + "_mask.exr")).string(); - ALICEVISION_LOG_TRACE("Load mask with path " << maskPath); - image::Image mask; - image::readImage(maskPath, mask, image::EImageColorSpace::NO_CONVERSION); - - //Compute list of intersection between this view and the reference view - std::vector intersections; - std::vector currentBoundingBoxes; - if (!panoramaMap->getIntersectionsList(intersections, currentBoundingBoxes, viewReference, viewCurrent)) - { - continue; - } - - for (int indexIntersection = 0; indexIntersection < intersections.size(); indexIntersection++) - { - const BoundingBox & bbox = currentBoundingBoxes[indexIntersection]; - const BoundingBox & bboxIntersect = intersections[indexIntersection]; - - BoundingBox cutBoundingBox; - cutBoundingBox.left = bboxIntersect.left - bbox.left; - cutBoundingBox.top = bboxIntersect.top - bbox.top; - cutBoundingBox.width = bboxIntersect.width; - cutBoundingBox.height = bboxIntersect.height; - if (cutBoundingBox.isEmpty()) - { - continue; - } - - - image::Image weights; - if (needWeights) - { - const std::string weightsPath = (fs::path(warpingFolder) / (std::to_string(viewCurrent) + "_weight.exr")).string(); - ALICEVISION_LOG_TRACE("Load weights with path " << weightsPath); - image::readImage(weightsPath, weights, image::EImageColorSpace::NO_CONVERSION); - } - else if (needSeams) - { - weights = image::Image(mask.Width(), mask.Height()); - if (!getMaskFromLabels(weights, labels, viewCurrent, bbox.left - referenceBoundingBox.left, bbox.top - referenceBoundingBox.top)) - { - ALICEVISION_LOG_ERROR("Error estimating seams image"); - return EXIT_FAILURE; - } - } - - image::Image subsource(cutBoundingBox.width, cutBoundingBox.height); - image::Image submask(cutBoundingBox.width, cutBoundingBox.height); - image::Image subweights(cutBoundingBox.width, cutBoundingBox.height); - - subsource = source.block(cutBoundingBox.top, cutBoundingBox.left, cutBoundingBox.height, cutBoundingBox.width); - submask = mask.block(cutBoundingBox.top, cutBoundingBox.left, cutBoundingBox.height, cutBoundingBox.width); - - if (weights.Width() > 0) - { - subweights = weights.block(cutBoundingBox.top, cutBoundingBox.left, cutBoundingBox.height, cutBoundingBox.width); - } - - if (!compositer->append(subsource, submask, subweights, bboxIntersect.left - referenceBoundingBox.left, bboxIntersect.top - referenceBoundingBox.top)) - { - ALICEVISION_LOG_INFO("Error in compositer append"); - return EXIT_FAILURE; - } - } - } - - - ALICEVISION_LOG_INFO("Terminate compositing for this view"); - if (!compositer->terminate()) - { - ALICEVISION_LOG_ERROR("Error terminating panorama"); - } - - const std::string viewIdStr = std::to_string(viewReference); - const std::string outputFilePath = (fs::path(outputFolder) / (viewIdStr + ".exr")).string(); - image::Image output = compositer->getOutput(); - - if (storageDataType == image::EStorageDataType::HalfFinite) - { - for (int i = 0; i < output.Height(); i++) - { - for (int j = 0; j < output.Width(); j++) - { - image::RGBAfColor ret; - image::RGBAfColor c = output(i, j); - - const float limit = float(HALF_MAX); - - ret.r() = clamp(c.r(), -limit, limit); - ret.g() = clamp(c.g(), -limit, limit); - ret.b() = clamp(c.b(), -limit, limit); - ret.a() = c.a(); - - output(i, j) = ret; - } - } + succeeded = false; + continue; } + } - oiio::ParamValueList metadata; - metadata.push_back(oiio::ParamValue("AliceVision:storageDataType", EStorageDataType_enumToString(storageDataType))); - metadata.push_back(oiio::ParamValue("AliceVision:offsetX", int(referenceBoundingBox.left))); - metadata.push_back(oiio::ParamValue("AliceVision:offsetY", int(referenceBoundingBox.top))); - metadata.push_back(oiio::ParamValue("AliceVision:panoramaWidth", int(panoramaMap->getWidth()))); - metadata.push_back(oiio::ParamValue("AliceVision:panoramaHeight", int(panoramaMap->getHeight()))); - - image::writeImage(outputFilePath, output, image::EImageColorSpace::LINEAR, metadata); + if (!succeeded) + { + return EXIT_FAILURE; } return EXIT_SUCCESS; From d70db9a3fe3324243618ac9d992c3c372723890c Mon Sep 17 00:00:00 2001 From: Fabien Date: Mon, 11 Jan 2021 11:07:56 +0100 Subject: [PATCH 56/79] [image] write IndexT images without intermediate float --- src/aliceVision/image/io.cpp | 140 ++++++++++++++++++++++++++--------- src/aliceVision/image/io.hpp | 1 + 2 files changed, 107 insertions(+), 34 deletions(-) diff --git a/src/aliceVision/image/io.cpp b/src/aliceVision/image/io.cpp index 7de21f739c..58793c2786 100644 --- a/src/aliceVision/image/io.cpp +++ b/src/aliceVision/image/io.cpp @@ -269,41 +269,18 @@ void readImage(const std::string& path, oiio::ImageBuf inBuf(path, 0, 0, NULL, &configSpec); - //Handle ROI option - oiio::ROI customROI = inBuf.roi(); - oiio::ROI subROI = imageReadOptions.subROI; - if (imageReadOptions.subROI.defined()) - { - //Make sure both roi are in the same frame - subROI.xbegin += inBuf.roi().xbegin; - subROI.ybegin += inBuf.roi().ybegin; - - //Keep the intersection - customROI = oiio::roi_intersection(subROI, customROI); - customROI.zbegin = inBuf.roi().zbegin; - customROI.zend = inBuf.roi().zend; - customROI.chbegin = inBuf.roi().chbegin; - customROI.chend = inBuf.roi().chend; - } - - inBuf.read(0, 0, false, oiio::TypeDesc::FLOAT); // force image convertion to float (for grayscale and color space convertion) + inBuf.read(0, 0, true, oiio::TypeDesc::FLOAT); // force image convertion to float (for grayscale and color space convertion) if(!inBuf.initialized()) - { throw std::runtime_error("Cannot find/open image file '" + path + "'."); - } // check picture channels number if(inBuf.spec().nchannels != 1 && inBuf.spec().nchannels < 3) - { throw std::runtime_error("Can't load channels of image file '" + path + "'."); - } // color conversion if(imageReadOptions.outputColorSpace == EImageColorSpace::AUTO) - { throw std::runtime_error("You must specify a requested color space for image file '" + path + "'."); - } const std::string& colorSpace = inBuf.spec().get_string_attribute("oiio:ColorSpace", "sRGB"); // default image color space is sRGB ALICEVISION_LOG_TRACE("Read image " << path << " (encoded in " << colorSpace << " colorspace)."); @@ -312,7 +289,7 @@ void readImage(const std::string& path, { if (colorSpace != "sRGB") { - oiio::ImageBufAlgo::colorconvert(inBuf, inBuf, colorSpace, "sRGB", true, "", "", nullptr, customROI); + oiio::ImageBufAlgo::colorconvert(inBuf, inBuf, colorSpace, "sRGB"); ALICEVISION_LOG_TRACE("Convert image " << path << " from " << colorSpace << " to sRGB colorspace"); } } @@ -320,7 +297,7 @@ void readImage(const std::string& path, { if (colorSpace != "Linear") { - oiio::ImageBufAlgo::colorconvert(inBuf, inBuf, colorSpace, "Linear", true, "", "", nullptr, customROI); + oiio::ImageBufAlgo::colorconvert(inBuf, inBuf, colorSpace, "Linear"); ALICEVISION_LOG_TRACE("Convert image " << path << " from " << colorSpace << " to Linear colorspace"); } } @@ -329,15 +306,15 @@ void readImage(const std::string& path, if(nchannels == 1 && inBuf.spec().nchannels >= 3) { // convertion region of interest (for inBuf.spec().nchannels > 3) - oiio::ROI convertionROI = customROI; - customROI.chbegin = 0; - customROI.chend = 3; + oiio::ROI convertionROI = inBuf.roi(); + convertionROI.chbegin = 0; + convertionROI.chend = 3; // compute luminance via a weighted sum of R,G,B // (assuming Rec709 primaries and a linear scale) const float weights[3] = {.2126f, .7152f, .0722f}; oiio::ImageBuf grayscaleBuf; - oiio::ImageBufAlgo::channel_sum(grayscaleBuf, inBuf, weights, customROI); + oiio::ImageBufAlgo::channel_sum(grayscaleBuf, inBuf, weights, convertionROI); inBuf.copy(grayscaleBuf); // TODO: if inBuf.spec().nchannels == 4: premult? @@ -369,9 +346,50 @@ void readImage(const std::string& path, } // copy pixels from oiio to eigen - image.resize(customROI.width(), customROI.height(), false); + image.resize(inBuf.spec().width, inBuf.spec().height, false); + { + oiio::ROI exportROI = inBuf.roi(); + exportROI.chbegin = 0; + exportROI.chend = nchannels; + + inBuf.get_pixels(exportROI, format, image.data()); + } +} + +void readImageIndexT(const std::string& path, + oiio::TypeDesc format, + int nchannels, + Image& image, + const ImageReadOptions & imageReadOptions) +{ + // check requested channels number + assert(nchannels == 1 || nchannels >= 3); + + oiio::ImageSpec configSpec; + + oiio::ImageBuf inBuf(path, 0, 0, NULL, &configSpec); + + inBuf.read(0, 0, true, oiio::TypeDesc::UINT32); // force image convertion to float (for grayscale and color space convertion) + + if(!inBuf.initialized()) + throw std::runtime_error("Cannot find/open image file '" + path + "'."); + + // check picture channels number + if(inBuf.spec().nchannels != 1 && inBuf.spec().nchannels < 3) + throw std::runtime_error("Can't load channels of image file '" + path + "'."); + + // color conversion + if(imageReadOptions.outputColorSpace == EImageColorSpace::AUTO) + throw std::runtime_error("You must specify a requested color space for image file '" + path + "'."); + + const std::string& colorSpace = inBuf.spec().get_string_attribute("oiio:ColorSpace", "sRGB"); // default image color space is sRGB + ALICEVISION_LOG_TRACE("Read image " << path << " (encoded in " << colorSpace << " colorspace)."); + + + // copy pixels from oiio to eigen + image.resize(inBuf.spec().width, inBuf.spec().height, false); { - oiio::ROI exportROI = customROI; + oiio::ROI exportROI = inBuf.roi(); exportROI.chbegin = 0; exportROI.chend = nchannels; @@ -426,7 +444,7 @@ void writeImage(const std::string& path, imageSpec.attribute("jpeg:subsampling", "4:4:4"); // if possible, always subsampling 4:4:4 for jpeg imageSpec.attribute("CompressionQuality", 100); // if possible, best compression quality - imageSpec.attribute("compression", isEXR ? "piz" : "none"); // if possible, set compression (piz for EXR, none for the other) + imageSpec.attribute("compression", isEXR ? "none" : "none"); // if possible, set compression (piz for EXR, none for the other) const oiio::ImageBuf imgBuf = oiio::ImageBuf(imageSpec, const_cast(image.data())); // original image buffer const oiio::ImageBuf* outBuf = &imgBuf; // buffer to write @@ -479,6 +497,55 @@ void writeImage(const std::string& path, fs::rename(tmpPath, path); } +void writeImageIndexT(const std::string& path, + oiio::TypeDesc typeDesc, + int nchannels, + const Image& image, + EImageColorSpace imageColorSpace, + const oiio::ParamValueList& metadata = oiio::ParamValueList()) +{ + const fs::path bPath = fs::path(path); + const std::string extension = boost::to_lower_copy(bPath.extension().string()); + const std::string tmpPath = (bPath.parent_path() / bPath.stem()).string() + "." + fs::unique_path().string() + extension; + const bool isEXR = (extension == ".exr"); + //const bool isTIF = (extension == ".tif"); + const bool isJPG = (extension == ".jpg"); + const bool isPNG = (extension == ".png"); + + if(imageColorSpace == EImageColorSpace::AUTO) + { + if(isJPG || isPNG) + imageColorSpace = EImageColorSpace::SRGB; + else + imageColorSpace = EImageColorSpace::LINEAR; + } + + oiio::ImageSpec imageSpec(image.Width(), image.Height(), nchannels, typeDesc); + imageSpec.extra_attribs = metadata; // add custom metadata + + imageSpec.attribute("jpeg:subsampling", "4:4:4"); // if possible, always subsampling 4:4:4 for jpeg + imageSpec.attribute("CompressionQuality", 100); // if possible, best compression quality + imageSpec.attribute("compression", isEXR ? "none" : "none"); // if possible, set compression (piz for EXR, none for the other) + + const oiio::ImageBuf imgBuf = oiio::ImageBuf(imageSpec, const_cast(image.data())); // original image buffer + const oiio::ImageBuf* outBuf = &imgBuf; // buffer to write + + oiio::ImageBuf formatBuf; // buffer for image format modification + if(isEXR) + { + + formatBuf.copy(*outBuf, oiio::TypeDesc::UINT32); // override format, use half instead of float + outBuf = &formatBuf; + } + + // write image + if(!outBuf->write(tmpPath)) + throw std::runtime_error("Can't write output image file '" + path + "'."); + + // rename temporay filename + fs::rename(tmpPath, path); +} + void readImage(const std::string& path, Image& image, const ImageReadOptions & imageReadOptions) { readImage(path, oiio::TypeDesc::FLOAT, 1, image, imageReadOptions); @@ -489,6 +556,11 @@ void readImage(const std::string& path, Image& image, const Image readImage(path, oiio::TypeDesc::UINT8, 1, image, imageReadOptions); } +void readImage(const std::string& path, Image& image, const ImageReadOptions & imageReadOptions) +{ + readImageIndexT(path, oiio::TypeDesc::UINT32, 1, image, imageReadOptions); +} + void readImage(const std::string& path, Image& image, const ImageReadOptions & imageReadOptions) { readImage(path, oiio::TypeDesc::FLOAT, 4, image, imageReadOptions); @@ -521,7 +593,7 @@ void writeImage(const std::string& path, const Image& image, EImageColorSpa void writeImage(const std::string& path, const Image& image, EImageColorSpace imageColorSpace, const oiio::ParamValueList& metadata) { - writeImage(path, oiio::TypeDesc::UINT32, 1, image, imageColorSpace, metadata); + writeImageIndexT(path, oiio::TypeDesc::UINT32, 1, image, imageColorSpace, metadata); } void writeImage(const std::string& path, const Image& image, EImageColorSpace imageColorSpace, const oiio::ParamValueList& metadata) diff --git a/src/aliceVision/image/io.hpp b/src/aliceVision/image/io.hpp index a0bbc5800a..fdc0133ff1 100644 --- a/src/aliceVision/image/io.hpp +++ b/src/aliceVision/image/io.hpp @@ -182,6 +182,7 @@ void getBufferFromImage(Image& image, oiio::ImageBuf& buffer); */ void readImage(const std::string& path, Image& image, const ImageReadOptions & imageReadOptions); void readImage(const std::string& path, Image& image, const ImageReadOptions & imageReadOptions); +void readImage(const std::string& path, Image& image, const ImageReadOptions & imageReadOptions); void readImage(const std::string& path, Image& image, const ImageReadOptions & imageReadOptions); void readImage(const std::string& path, Image& image, const ImageReadOptions & imageReadOptions); void readImage(const std::string& path, Image& image, const ImageReadOptions & imageReadOptions); From 3377211c2b3190c01b33e81e922ceb89f71701c2 Mon Sep 17 00:00:00 2001 From: Fabien Date: Mon, 11 Jan 2021 11:08:29 +0100 Subject: [PATCH 57/79] [panorama] code convention --- src/aliceVision/panorama/alphaCompositer.hpp | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/src/aliceVision/panorama/alphaCompositer.hpp b/src/aliceVision/panorama/alphaCompositer.hpp index 460268e601..ff2c5996b3 100644 --- a/src/aliceVision/panorama/alphaCompositer.hpp +++ b/src/aliceVision/panorama/alphaCompositer.hpp @@ -13,20 +13,23 @@ class AlphaCompositer : public Compositer { } - virtual bool append(aliceVision::image::Image& color, - aliceVision::image::Image& inputMask, - aliceVision::image::Image& inputWeights, - int offset_x, int offset_y) + virtual bool append(const aliceVision::image::Image& color, + const aliceVision::image::Image& inputMask, + const aliceVision::image::Image& inputWeights, + int offsetX, int offsetY) { + offsetX -= _outputRoi.left; + offsetY -= _outputRoi.top; + for(int i = 0; i < color.Height(); i++) { - int y = i + offset_y; - if (y < 0 || y >= _panoramaHeight) continue; + int y = i + offsetY; + if (y < 0 || y >= _outputRoi.height) continue; for (int j = 0; j < color.Width(); j++) { - int x = j + offset_x; - if (x < 0 || x >= _panoramaWidth) continue; + int x = j + offsetX; + if (x < 0 || x >= _outputRoi.width) continue; if (!inputMask(i, j)) { From 613671c2dd07f04fb054d6a3a92144e2ccb1d342 Mon Sep 17 00:00:00 2001 From: Fabien Date: Mon, 11 Jan 2021 11:08:56 +0100 Subject: [PATCH 58/79] [sfmdata] correction in equation in comments --- src/aliceVision/sfmData/View.cpp | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/aliceVision/sfmData/View.cpp b/src/aliceVision/sfmData/View.cpp index 14cc3d4bae..76e31fbdb2 100644 --- a/src/aliceVision/sfmData/View.cpp +++ b/src/aliceVision/sfmData/View.cpp @@ -14,6 +14,9 @@ #include +#include + + namespace aliceVision { namespace sfmData { @@ -33,10 +36,12 @@ float View::getCameraExposureSetting(const float referenceISO, const float refer const float iso = getMetadataISO(); /* iso = qLt / aperture^2 - isoratio = iso2 / iso1 = (qLt / aperture1^2) / (qLt / aperture2^2) - isoratio = aperture2^2 / aperture1^2 - aperture2^2 = iso2 / iso1 * 1 - aperture2 = sqrt(iso2 / iso1) + isoratio = iso2 / iso1 = (qLt / aperture2^2) / (qLt / aperture1^2) + isoratio = aperture1^2 / aperture2^2 + aperture2^2 = aperture1^2 / isoratio + aperture2^2 = (aperture1^2 / (iso2 / iso1)) + aperture2^2 = (iso1 / iso2) + aperture2 = sqrt(iso1 / iso2) */ float iso_2_aperture = 1.0f; if(iso > 1e-6f && referenceISO > 1e-6f) @@ -47,8 +52,9 @@ float View::getCameraExposureSetting(const float referenceISO, const float refer /* aperture = f / diameter - aperture2 / aperture1 = diameter2 / diameter1 - (aperture2 / aperture1)^2 = (area2 / pi) / (area1 / pi) + aperture2 / aperture1 = diameter1 / diameter2 + (aperture2 / aperture1)^2 = (area1 / pi) / (area2 / pi) + area2 = (aperture1 / aperture2)^2 */ float new_fnumber = fnumber * iso_2_aperture; float exp_increase = (new_fnumber / lReferenceFNumber) * (new_fnumber / lReferenceFNumber); From c6c3b851dccc569dc7cff92fea245c46b07d2fc2 Mon Sep 17 00:00:00 2001 From: Fabien Date: Mon, 11 Jan 2021 11:10:03 +0100 Subject: [PATCH 59/79] [panorama] module for panorama seams estimation --- src/aliceVision/panorama/seams.cpp | 258 +++++-------------- src/aliceVision/panorama/seams.hpp | 16 +- src/software/pipeline/main_panoramaSeams.cpp | 105 +++----- 3 files changed, 108 insertions(+), 271 deletions(-) diff --git a/src/aliceVision/panorama/seams.cpp b/src/aliceVision/panorama/seams.cpp index a011ef0c93..b4330404a9 100644 --- a/src/aliceVision/panorama/seams.cpp +++ b/src/aliceVision/panorama/seams.cpp @@ -305,9 +305,62 @@ bool WTASeams::append(const aliceVision::image::Image& inputMask, return true; } -bool HierarchicalGraphcutSeams::setOriginalLabels(CachedImage& labels) +bool WTASeams::appendWithLoop(const aliceVision::image::Image& inputMask, + const aliceVision::image::Image& inputWeights, + IndexT currentIndex, size_t offset_x, size_t offset_y) +{ + if (inputMask.size() != inputWeights.size()) + { + return false; + } + + for (size_t i = 0; i < inputWeights.Height(); i++) + { + int y = i + offset_y; + if (y < 0 || y >= _panoramaHeight) continue; + + for (size_t j = 0; j < inputWeights.Width(); j++) + { + int x = j + offset_x; + if (x < 0) + { + x += _panoramaWidth; + } + if (x >= _panoramaWidth) + { + x -= _panoramaWidth; + } + + if (!inputMask(i, j)) + { + continue; + } + + if (inputWeights(i, j) > _weights(y, x)) + { + _labels(y, x) = currentIndex; + _weights(y, x) = inputWeights(i, j); + } + } + } + + return true; +} + +bool HierarchicalGraphcutSeams::initialize(const image::Image & labels) { - + int width = _outputWidth; + int height = _outputHeight; + + for (int i = 0; i < _countLevels; i++) + { + _graphcuts.emplace_back(width, height); + + //Divide by 2 (rounding to the superior integer) + width = int(ceil(float(width) / 2.0f)); + height = int(ceil(float(height) / 2.0f)); + } + if (!_graphcuts[0].setOriginalLabels(labels)) { return false; @@ -321,56 +374,12 @@ bool HierarchicalGraphcutSeams::setOriginalLabels(CachedImage& labels) for (int l = 1; l < _countLevels; l++) { - CachedImage & largerLabels = _graphcuts[l - 1].getLabels(); - CachedImage & smallerLabels = _graphcuts[l].getLabels(); + image::Image & largerLabels = _graphcuts[l - 1].getLabels(); + image::Image & smallerLabels = _graphcuts[l].getLabels(); - int processingSize = 256; - - for (int i = 0; i < smallerLabels.getHeight(); i+= processingSize) - { - for (int j = 0; j < smallerLabels.getWidth(); j+= processingSize) - { - BoundingBox smallBb; - smallBb.left = j; - smallBb.top = i; - smallBb.width = processingSize; - smallBb.height = processingSize; - smallBb.clampRight(smallerLabels.getWidth() - 1); - smallBb.clampBottom(smallerLabels.getHeight() - 1); - - BoundingBox smallInputBb; - smallInputBb.left = 0; - smallInputBb.top = 0; - smallInputBb.width = smallBb.width; - smallInputBb.height = smallBb.height; - - BoundingBox largeBb = smallBb.doubleSize(); - largeBb.clampRight(largerLabels.getWidth() - 1); - largeBb.clampBottom(largerLabels.getHeight() - 1); - - BoundingBox largeInputBb; - largeInputBb.left = 0; - largeInputBb.top = 0; - largeInputBb.width = largeBb.width; - largeInputBb.height = largeBb.height; - - image::Image largeView(largeBb.width, largeBb.height); - if (!largerLabels.extract(largeView, largeInputBb, largeBb)) - { - return false; - } - - image::Image smallView(smallBb.width, smallBb.height); - downscale(smallView, largeView); - - if (!smallerLabels.assign(smallView, smallInputBb, smallBb)) - { - return false; - } - } - } + downscale(smallerLabels, largerLabels); } - + return true; } @@ -397,48 +406,8 @@ bool HierarchicalGraphcutSeams::append(const aliceVision::image::Image destColor; - if (!destColor.createImage(_cacheManager, width, height)) - { - return false; - } - - if (!destColor.fill(image::RGBfColor(0.0f))) - { - return false; - } - - if (!destColor.assign(feathered, bb, bb)) - { - return false; - } - - //Create cached mask and assign content - CachedImage destMask; - if (!destMask.createImage(_cacheManager, width, height)) - { - return false; - } - - if (!destMask.fill(0)) - { - return false; - } - - if (!destMask.assign(potMask, bb, bb)) - { - return false; - } - //Assign content to graphcut - if (!_graphcuts[level].append(destColor, destMask, currentIndex, newOffsetX, newOffsetY)) + if (!_graphcuts[level].append(feathered, potMask, currentIndex, newOffsetX, newOffsetY)) { return false; } @@ -473,21 +442,6 @@ bool HierarchicalGraphcutSeams::append(const aliceVision::image::Image & smallLabels = _graphcuts[level].getLabels(); - int w = smallLabels.getWidth(); - int h = smallLabels.getHeight(); + image::Image & smallLabels = _graphcuts[level].getLabels(); + int w = smallLabels.Width(); + int h = smallLabels.Height(); if (level == _countLevels - 1) { @@ -528,13 +482,8 @@ bool HierarchicalGraphcutSeams::process() _graphcuts[level].setMaximalDistance(200); } - bool computeAlphaExpansion = false; - if (w < 5000) - { - computeAlphaExpansion = true; - } - if(!_graphcuts[level].process(computeAlphaExpansion)) + if(!_graphcuts[level].process()) { return false; } @@ -545,57 +494,16 @@ bool HierarchicalGraphcutSeams::process() } //Enlarge result of this level to be an initialization for next level - CachedImage & largeLabels = _graphcuts[level - 1].getLabels(); + image::Image & largeLabels = _graphcuts[level - 1].getLabels(); - const int processingSize = 256; - - for (int i = 0; i < smallLabels.getHeight(); i+= processingSize) + for (int y = 0; y < largeLabels.Height(); y++) { - for (int j = 0; j < smallLabels.getWidth(); j+= processingSize) + int hy = y / 2; + + for (int x = 0; x < largeLabels.Width(); x++) { - BoundingBox smallBb; - smallBb.left = j; - smallBb.top = i; - smallBb.width = processingSize; - smallBb.height = processingSize; - smallBb.clampRight(smallLabels.getWidth() - 1); - smallBb.clampBottom(smallLabels.getHeight() - 1); - - BoundingBox smallInputBb = smallBb; - smallInputBb.left = 0; - smallInputBb.top = 0; - - - image::Image smallView(smallBb.width, smallBb.height); - if (!smallLabels.extract(smallView, smallInputBb, smallBb)) - { - return false; - } - - BoundingBox largeBb = smallBb.doubleSize(); - largeBb.clampRight(largeLabels.getWidth() - 1); - largeBb.clampBottom(largeLabels.getHeight() - 1); - - BoundingBox largeInputBb = largeBb; - largeInputBb.left = 0; - largeInputBb.top = 0; - - image::Image largeView(largeBb.width, largeBb.height); - for (int y = 0; y < largeBb.height; y++) - { - int hy = y / 2; - - for (int x = 0; x < largeBb.width; x++) - { - int hx = x / 2; - largeView(y, x) = smallView(hy, hx); - } - } - - if (!largeLabels.assign(largeView, largeInputBb, largeBb)) - { - return false; - } + int hx = x / 2; + largeLabels(y, x) = smallLabels(hy, hx); } } } @@ -604,30 +512,6 @@ bool HierarchicalGraphcutSeams::process() return true; } -bool HierarchicalGraphcutSeams::initialize() -{ - int width = _outputWidth; - int height = _outputHeight; - - for (int i = 0; i < _countLevels; i++) - { - GraphcutSeams gcs(width, height); - - if (!gcs.initialize(_cacheManager)) - { - return false; - } - - _graphcuts.push_back(gcs); - - //Divide by 2 (rounding to the superior integer) - width = int(ceil(float(width) / 2.0f)); - height = int(ceil(float(height) / 2.0f)); - } - - return true; -} - bool getMaskFromLabels(aliceVision::image::Image & mask, image::Image & labels, IndexT index, int offset_x, int offset_y) { diff --git a/src/aliceVision/panorama/seams.hpp b/src/aliceVision/panorama/seams.hpp index 7f41c4ea09..4af031747a 100644 --- a/src/aliceVision/panorama/seams.hpp +++ b/src/aliceVision/panorama/seams.hpp @@ -31,6 +31,10 @@ class WTASeams const aliceVision::image::Image& inputWeights, IndexT currentIndex, size_t offset_x, size_t offset_y); + bool appendWithLoop(const aliceVision::image::Image& inputMask, + const aliceVision::image::Image& inputWeights, + IndexT currentIndex, size_t offset_x, size_t offset_y); + image::Image & getLabels() { return _labels; @@ -47,9 +51,8 @@ class WTASeams class HierarchicalGraphcutSeams { public: - HierarchicalGraphcutSeams(image::TileCacheManager::shared_ptr cacheManager, size_t outputWidth, size_t outputHeight, size_t countLevels) - : _cacheManager(cacheManager) - , _outputWidth(outputWidth) + HierarchicalGraphcutSeams(size_t outputWidth, size_t outputHeight, size_t countLevels) + : _outputWidth(outputWidth) , _outputHeight(outputHeight) , _countLevels(countLevels) { @@ -57,9 +60,7 @@ class HierarchicalGraphcutSeams virtual ~HierarchicalGraphcutSeams() = default; - bool initialize(); - - bool setOriginalLabels(CachedImage& labels); + bool initialize(const image::Image& labels); virtual bool append(const aliceVision::image::Image& input, const aliceVision::image::Image& inputMask, @@ -67,14 +68,13 @@ class HierarchicalGraphcutSeams bool process(); - CachedImage& getLabels() + image::Image& getLabels() { return _graphcuts[0].getLabels(); } private: std::vector _graphcuts; - image::TileCacheManager::shared_ptr _cacheManager; size_t _countLevels; size_t _outputWidth; diff --git a/src/software/pipeline/main_panoramaSeams.cpp b/src/software/pipeline/main_panoramaSeams.cpp index 431ed05355..fd0d622c0e 100644 --- a/src/software/pipeline/main_panoramaSeams.cpp +++ b/src/software/pipeline/main_panoramaSeams.cpp @@ -69,7 +69,7 @@ bool computeWTALabels(image::Image & labels, const std::vector weights; image::readImage(weightsPath, weights, image::EImageColorSpace::NO_CONVERSION); - if (!seams.append(mask, weights, viewId, offsetX, offsetY)) + if (!seams.appendWithLoop(mask, weights, viewId, offsetX, offsetY)) { return false; } @@ -80,22 +80,16 @@ bool computeWTALabels(image::Image & labels, const std::vector & labels, image::TileCacheManager::shared_ptr& cacheManager, const std::vector> & views, const std::string & inputPath, std::pair & panoramaSize, int smallestViewScale) +bool computeGCLabels(image::Image & labels, const std::vector> & views, const std::string & inputPath, std::pair & panoramaSize, int smallestViewScale) { ALICEVISION_LOG_INFO("Estimating smart seams for panorama"); int pyramidSize = 1 + std::max(0, smallestViewScale - 1); ALICEVISION_LOG_INFO("Graphcut pyramid size is " << pyramidSize); - HierarchicalGraphcutSeams seams(cacheManager, panoramaSize.first, panoramaSize.second, pyramidSize); + HierarchicalGraphcutSeams seams(panoramaSize.first, panoramaSize.second, pyramidSize); - - if (!seams.initialize()) - { - return false; - } - - if (!seams.setOriginalLabels(labels)) + if (!seams.initialize(labels)) { return false; } @@ -181,8 +175,7 @@ int aliceVision_main(int argc, char** argv) requiredParams.add_options() ("input,i", po::value(&sfmDataFilepath)->required(), "Input sfmData.") ("warpingFolder,w", po::value(&warpingFolder)->required(), "Folder with warped images.") - ("output,o", po::value(&outputLabels)->required(), "Path of the output labels.") - ("cacheFolder,f", po::value(&temporaryCachePath)->required(), "Path of the temporary cache."); + ("output,o", po::value(&outputLabels)->required(), "Path of the output labels."); allParams.add(requiredParams); // Description of optional parameters @@ -266,91 +259,51 @@ int aliceVision_main(int argc, char** argv) ALICEVISION_LOG_INFO("Output labels size set to " << panoramaSize.first << "x" << panoramaSize.second); } - if(!temporaryCachePath.empty() && !fs::exists(temporaryCachePath)) - { - fs::create_directory(temporaryCachePath); - } - - // Create a cache manager - image::TileCacheManager::shared_ptr cacheManager = image::TileCacheManager::create(temporaryCachePath, 256, 256, 65536); - if(!cacheManager) - { - ALICEVISION_LOG_ERROR("Error creating the cache manager"); - return EXIT_FAILURE; - } - - // Configure the cache manager memory - system::MemoryInfo memInfo = system::getMemoryInfo(); - const double convertionGb = std::pow(2,30); - ALICEVISION_LOG_INFO("Available RAM is " << std::setw(5) << memInfo.availableRam / convertionGb << "GB (" << memInfo.availableRam << " octets)."); - cacheManager->setMaxMemory(1024ll * 1024ll * 1024ll * 6ll); - - //Get a list of views ordered by their image scale - size_t smallestScale = 0; - std::vector> viewOrderedByScale; + int smallestScale = 10000; + std::vector> views; + for (const auto & it : sfmData.getViews()) { - std::map>> mapViewsScale; - for(const auto & it : sfmData.getViews()) - { - auto view = it.second; - IndexT viewId = view->getViewId(); - - if(!sfmData.isPoseAndIntrinsicDefined(view.get())) - { - // skip unreconstructed views - continue; - } - - // Load mask - const std::string maskPath = (fs::path(warpingFolder) / (std::to_string(viewId) + "_mask.exr")).string(); - image::Image mask; - image::readImage(maskPath, mask, image::EImageColorSpace::NO_CONVERSION); - - //Estimate scale - size_t scale = getGraphcutOptimalScale(mask.Width(), mask.Height()); - mapViewsScale[scale].push_back(it.second); - } - - if (mapViewsScale.size() == 0) - { - ALICEVISION_LOG_ERROR("No valid view"); - return EXIT_FAILURE; - } + auto view = it.second; + IndexT viewId = view->getViewId(); - smallestScale = mapViewsScale.begin()->first; - for (auto scaledList : mapViewsScale) + if(!sfmData.isPoseAndIntrinsicDefined(view.get())) { - for (auto item : scaledList.second) - { - viewOrderedByScale.push_back(item); - } + // skip unreconstructed views + continue; } + + // Load mask + const std::string maskPath = (fs::path(warpingFolder) / (std::to_string(viewId) + "_mask.exr")).string(); + int width, height; + image::readImageMetadata(maskPath, width, height); + + //Estimate scale + int scale = getGraphcutOptimalScale(width, height); + + smallestScale = std::min(scale, smallestScale); + views.push_back(view); } - ALICEVISION_LOG_INFO(viewOrderedByScale.size() << " views to process"); + ALICEVISION_LOG_INFO(views.size() << " views to process"); image::Image labels; - if (!computeWTALabels(labels, viewOrderedByScale, warpingFolder, panoramaSize)) + if (!computeWTALabels(labels, views, warpingFolder, panoramaSize)) { ALICEVISION_LOG_ERROR("Error computing initial labels"); return EXIT_FAILURE; } - /*if (useGraphCut) + if (useGraphCut) { - if (!computeGCLabels(labels, cacheManager, viewOrderedByScale, warpingFolder, panoramaSize, smallestScale)) + if (!computeGCLabels(labels, views, warpingFolder, panoramaSize, smallestScale)) { ALICEVISION_LOG_ERROR("Error computing graph cut labels"); return EXIT_FAILURE; } } - if (!labels.writeImage(outputLabels)) - { - ALICEVISION_LOG_ERROR("Error writing labels to disk"); - return EXIT_FAILURE; - }*/ + image::writeImage(outputLabels, labels, image::EImageColorSpace::NO_CONVERSION); return EXIT_SUCCESS; } From 82807c9f177eab1ad5f310873f5fe010cf5e2cf2 Mon Sep 17 00:00:00 2001 From: Fabien Date: Mon, 11 Jan 2021 11:10:35 +0100 Subject: [PATCH 60/79] [panorama] bounding box multiply and divide correctly using scale instead of factor --- src/aliceVision/panorama/boundingBox.hpp | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/src/aliceVision/panorama/boundingBox.hpp b/src/aliceVision/panorama/boundingBox.hpp index b71e057c4f..0173fe696e 100644 --- a/src/aliceVision/panorama/boundingBox.hpp +++ b/src/aliceVision/panorama/boundingBox.hpp @@ -183,10 +183,12 @@ struct BoundingBox return b; } - BoundingBox multiply(int factor) const + BoundingBox multiply(int scale) const { BoundingBox b; + int factor = pow(2, scale); + b.left = left * factor; b.top = top * factor; b.width = width * factor; @@ -195,18 +197,23 @@ struct BoundingBox return b; } - BoundingBox divide(int factor) const + BoundingBox divide(int scale) const { BoundingBox b; + int factor = pow(2, scale); + b.left = int(floor(double(left) / double(factor))); b.top = int(floor(double(top) / double(factor))); - int right = int(ceil(double(getRight()) / double(factor))); - int bottom = int(ceil(double(getBottom()) / double(factor))); + int sleft = b.left * factor; + int stop = b.top * factor; - b.width = right - b.left + 1; - b.height = bottom - b.top + 1; + int nwidth = getRight() - sleft; + int nheight = getBottom() - stop; + + b.width = int(ceil(double(nwidth) / double(factor))); + b.height = int(ceil(double(nheight) / double(factor))); return b; } From 3a90af595f066cb052260e7d35d2055aaee7d7f2 Mon Sep 17 00:00:00 2001 From: Fabien Date: Mon, 11 Jan 2021 11:11:02 +0100 Subject: [PATCH 61/79] [panorama] compositer have output roi --- src/aliceVision/panorama/compositer.hpp | 31 +++++++++++++++++-------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/src/aliceVision/panorama/compositer.hpp b/src/aliceVision/panorama/compositer.hpp index 4a63aed29d..fb3ff182b3 100644 --- a/src/aliceVision/panorama/compositer.hpp +++ b/src/aliceVision/panorama/compositer.hpp @@ -24,20 +24,23 @@ class Compositer } - virtual bool append(aliceVision::image::Image& color, - aliceVision::image::Image& inputMask, - aliceVision::image::Image& inputWeights, - int offset_x, int offset_y) + virtual bool append(const aliceVision::image::Image& color, + const aliceVision::image::Image& inputMask, + const aliceVision::image::Image& inputWeights, + int offsetX, int offsetY) { + offsetX -= _outputRoi.left; + offsetY -= _outputRoi.top; + for(int i = 0; i < color.Height(); i++) { - int y = i + offset_y; - if (y < 0 || y >= _panoramaHeight) continue; + int y = i + offsetY; + if (y < 0 || y >= _outputRoi.height) continue; for (int j = 0; j < color.Width(); j++) { - int x = j + offset_x; - if (x < 0 || x >= _panoramaWidth) continue; + int x = j + offsetX; + if (x < 0 || x >= _outputRoi.width) continue; if (!inputMask(i, j)) { @@ -54,9 +57,16 @@ class Compositer return true; } - virtual bool initialize() { + virtual bool initialize(const BoundingBox & outputRoi) { + + _outputRoi = outputRoi; + + if (_outputRoi.left < 0) return false; + if (_outputRoi.top < 0) return false; + if (_outputRoi.getRight() >= _panoramaWidth) return false; + if (_outputRoi.getBottom() >= _panoramaHeight) return false; - _panorama = image::Image(_panoramaWidth, _panoramaHeight, true, image::RGBAfColor(0.0f, 0.0f, 0.0f, 0.0f)); + _panorama = image::Image(_outputRoi.width, _outputRoi.height, true, image::RGBAfColor(0.0f, 0.0f, 0.0f, 0.0f)); return true; } @@ -78,6 +88,7 @@ class Compositer image::Image _panorama; int _panoramaWidth; int _panoramaHeight; + BoundingBox _outputRoi; }; } // namespace aliceVision From 33be69a7a420c6443f2979ec58c913180519803d Mon Sep 17 00:00:00 2001 From: Fabien Date: Mon, 11 Jan 2021 11:11:37 +0100 Subject: [PATCH 62/79] [panorama] new way of handling large images --- src/aliceVision/panorama/graphcut.hpp | 99 ++--- src/aliceVision/panorama/imageOps.hpp | 93 +++++ .../panorama/laplacianCompositer.hpp | 35 +- src/aliceVision/panorama/laplacianPyramid.cpp | 20 +- src/aliceVision/panorama/laplacianPyramid.hpp | 2 +- src/aliceVision/panorama/panoramaMap.cpp | 52 ++- src/aliceVision/panorama/panoramaMap.hpp | 10 +- .../pipeline/main_panoramaCompositing.cpp | 369 ++++++++++++------ 8 files changed, 449 insertions(+), 231 deletions(-) diff --git a/src/aliceVision/panorama/graphcut.hpp b/src/aliceVision/panorama/graphcut.hpp index 1122345ade..2a7fbb524d 100644 --- a/src/aliceVision/panorama/graphcut.hpp +++ b/src/aliceVision/panorama/graphcut.hpp @@ -142,8 +142,8 @@ class GraphcutSeams { IndexT id; BoundingBox rect; - CachedImage color; - CachedImage mask; + image::Image color; + image::Image mask; }; using IndexedColor = std::pair; @@ -152,8 +152,10 @@ class GraphcutSeams public: GraphcutSeams(size_t outputWidth, size_t outputHeight) - : _outputWidth(outputWidth), _outputHeight(outputHeight) + : _outputWidth(outputWidth) + , _outputHeight(outputHeight) , _maximal_distance_change(outputWidth + outputHeight) + , _labels(outputWidth, outputHeight, true, UndefinedIndexT) { } @@ -185,49 +187,36 @@ class GraphcutSeams return false; } - bool setOriginalLabels(CachedImage & existing_labels) + bool setOriginalLabels(const image::Image & existing_labels) { - if (!_labels.deepCopy(existing_labels)) - { - return false; - } - - return true; - } - - bool initialize(image::TileCacheManager::shared_ptr & cacheManager) - { - if(!_labels.createImage(cacheManager, _outputWidth, _outputHeight)) - { - return false; - } + _labels = existing_labels; return true; } - bool append(const CachedImage& input, const CachedImage& inputMask, IndexT currentIndex, size_t offset_x, size_t offset_y) + bool append(const image::Image& input, const image::Image& inputMask, IndexT currentIndex, size_t offset_x, size_t offset_y) { - if(inputMask.getWidth() != input.getWidth()) + if(inputMask.Width() != input.Width()) { return false; } - if(inputMask.getHeight() != input.getHeight()) + if(inputMask.Height() != input.Height()) { return false; } InputData data; + data.id = currentIndex; data.color = input; data.mask = inputMask; - data.rect.width = input.getWidth(); - data.rect.height = input.getHeight(); + data.rect.width = input.Width(); + data.rect.height = input.Height(); data.rect.left = offset_x; data.rect.top = offset_y; - _inputs[currentIndex] = data; @@ -239,7 +228,7 @@ class GraphcutSeams _maximal_distance_change = dist; } - bool createInputOverlappingObservations(image::Image & graphCutInput, const BoundingBox & interestBbox, bool loadColors) + bool createInputOverlappingObservations(image::Image & graphCutInput, const BoundingBox & interestBbox) { for (auto & otherInput : _inputs) { @@ -249,7 +238,6 @@ class GraphcutSeams BoundingBox otherBboxLoopRight = otherInput.second.rect; otherBboxLoopRight.left = otherBbox.left + _outputWidth; - BoundingBox otherInputBbox = otherBbox; otherInputBbox.left = 0; otherInputBbox.top = 0; @@ -263,20 +251,11 @@ class GraphcutSeams } image::Image otherColor(otherInputBbox.width, otherInputBbox.height); - if (loadColors) - { - if (!otherInput.second.color.extract(otherColor, otherInputBbox, otherInputBbox)) - { - return false; - } - } + otherColor.block(otherInputBbox.top, otherInputBbox.left, otherInputBbox.height, otherInputBbox.width) = otherInput.second.color.block(otherInputBbox.top, otherInputBbox.left, otherInputBbox.height, otherInputBbox.width); image::Image otherMask(otherInputBbox.width, otherInputBbox.height); - if (!otherInput.second.mask.extract(otherMask, otherInputBbox, otherInputBbox)) - { - return false; - } - + otherMask.block(otherInputBbox.top, otherInputBbox.left, otherInputBbox.height, otherInputBbox.width) = otherInput.second.mask.block(otherInputBbox.top, otherInputBbox.left, otherInputBbox.height, otherInputBbox.width); + if (!intersection.isEmpty()) { BoundingBox interestOther = intersection; @@ -495,7 +474,7 @@ class GraphcutSeams return true; } - bool processInput(double & newCost, InputData & input, bool computeExpansion) + bool processInput(double & newCost, InputData & input) { //Get bounding box of input in panorama @@ -503,7 +482,7 @@ class GraphcutSeams BoundingBox localBbox = input.rect.dilate(3); localBbox.clampLeft(); localBbox.clampTop(); - localBbox.clampBottom(_labels.getHeight() - 1); + localBbox.clampBottom(_labels.Height() - 1); //Output must keep a margin also BoundingBox outputBbox = input.rect; @@ -512,15 +491,14 @@ class GraphcutSeams //Extract labels for the ROI image::Image localLabels(localBbox.width, localBbox.height); - if (!loopyCachedImageExtract(localLabels, _labels, localBbox)) + if (!loopyImageExtract(localLabels, _labels, localBbox)) { return false; } - //Build the input image::Image graphCutInput(localBbox.width, localBbox.height, true); - if (!createInputOverlappingObservations(graphCutInput, localBbox, computeExpansion)) + if (!createInputOverlappingObservations(graphCutInput, localBbox)) { return false; } @@ -535,16 +513,11 @@ class GraphcutSeams BoundingBox inputBb = localBbox; inputBb.left = 0; inputBb.top = 0; - if (!loopyCachedImageAssign(_labels, localLabels, localBbox, inputBb)) + if (!loopyImageAssign(_labels, localLabels, localBbox, inputBb)) { return false; } - if (!computeExpansion) - { - return true; - } - // Compute distance map to borders of the input seams image::Image distanceMap(localLabels.Width(), localLabels.Height()); if (!computeInputDistanceMap(distanceMap, localLabels, input.id)) @@ -581,7 +554,7 @@ class GraphcutSeams inputBb.left = 0; inputBb.top = 0; - if (!loopyCachedImageAssign(_labels, localLabels, localBbox, inputBb)) + if (!loopyImageAssign(_labels, localLabels, localBbox, inputBb)) { return false; } @@ -594,24 +567,8 @@ class GraphcutSeams return true; } - bool process(bool computeAlphaExpansion) + bool process() { - if (!computeAlphaExpansion) - { - for (auto & info : _inputs) - { - double cost; - ALICEVISION_LOG_INFO("process input"); - - if (!processInput(cost, info.second, false)) - { - return false; - } - } - - return true; - } - std::map costs; for (auto & info : _inputs) { @@ -621,6 +578,7 @@ class GraphcutSeams for (int i = 0; i < 10; i++) { ALICEVISION_LOG_INFO("GraphCut processing iteration #" << i); + // For each possible label, try to extends its domination on the label's world bool hasChange = false; @@ -628,10 +586,11 @@ class GraphcutSeams for (auto & info : _inputs) { double cost; - if (!processInput(cost, info.second, true)) + if (!processInput(cost, info.second)) { return false; } + if (costs[info.first] != cost) { @@ -1030,7 +989,7 @@ class GraphcutSeams return true; } - CachedImage & getLabels() + image::Image & getLabels() { return _labels; } @@ -1042,7 +1001,7 @@ class GraphcutSeams int _outputWidth; int _outputHeight; size_t _maximal_distance_change; - CachedImage _labels; + image::Image _labels; }; } // namespace aliceVision diff --git a/src/aliceVision/panorama/imageOps.hpp b/src/aliceVision/panorama/imageOps.hpp index 90d98c4858..624f027d2e 100644 --- a/src/aliceVision/panorama/imageOps.hpp +++ b/src/aliceVision/panorama/imageOps.hpp @@ -165,6 +165,55 @@ bool addition(aliceVision::image::Image& AplusB, const aliceVision::image::Im void removeNegativeValues(image::Image& img); +template +bool loopyImageAssign(image::Image & output, const aliceVision::image::Image & input, const BoundingBox & assignedOutputBb, const BoundingBox & assignedInputBb) +{ + BoundingBox inputBb = assignedInputBb; + BoundingBox outputBb = assignedOutputBb; + + if (inputBb.width != outputBb.width) + { + return false; + } + + if (inputBb.height != outputBb.height) + { + return false; + } + + if (assignedOutputBb.getRight() < output.Width()) + { + output.block(outputBb.top, outputBb.left, outputBb.height, outputBb.width) = input.block(inputBb.top, inputBb.left, inputBb.height, inputBb.width); + } + else + { + int left_1 = assignedOutputBb.left; + int left_2 = 0; + int width1 = output.Width() - assignedOutputBb.left; + int width2 = input.Width() - width1; + + inputBb.left = 0; + outputBb.left = left_1; + inputBb.width = width1; + outputBb.width = width1; + + output.block(outputBb.top, outputBb.left, outputBb.height, outputBb.width) = input.block(inputBb.top, inputBb.left, inputBb.height, inputBb.width); + + inputBb.left = width1; + outputBb.left = 0; + + //no overlap + int width2_clamped = std::min(width2, left_1); + inputBb.width = width2_clamped; + outputBb.width = width2_clamped; + if (width2_clamped == 0) return true; + + output.block(outputBb.top, outputBb.left, outputBb.height, outputBb.width) = input.block(inputBb.top, inputBb.left, inputBb.height, inputBb.width); + } + + return true; +} + template bool loopyCachedImageAssign(CachedImage & output, const aliceVision::image::Image & input, const BoundingBox & assignedOutputBb, const BoundingBox & assignedInputBb) { @@ -226,6 +275,50 @@ bool loopyCachedImageAssign(CachedImage & output, const aliceVision::image::I return true; } +template +bool loopyImageExtract(image::Image & output, const image::Image & input, const BoundingBox & extractedInputBb) +{ + BoundingBox outputBb; + BoundingBox inputBb; + + outputBb.left = 0; + outputBb.top = 0; + outputBb.width = output.Width(); + outputBb.height = std::min(extractedInputBb.height, output.Height()); + + inputBb = extractedInputBb; + + if (inputBb.getRight() < input.Width()) + { + output.block(outputBb.top, outputBb.left, outputBb.height, outputBb.width) = input.block(inputBb.top, inputBb.left, inputBb.height, inputBb.width); + } + else + { + int availableWidth = output.Width(); + while (availableWidth > 0) + { + inputBb.clampRight(input.Width() - 1); + int extractedWidth = std::min(inputBb.width, availableWidth); + + + inputBb.width = extractedWidth; + outputBb.width = extractedWidth; + + output.block(outputBb.top, outputBb.left, outputBb.height, outputBb.width) = input.block(inputBb.top, inputBb.left, inputBb.height, inputBb.width); + + //Update the bouding box for output + outputBb.left += extractedWidth; + availableWidth -= extractedWidth; + + //All the input is available. + inputBb.left = 0; + inputBb.width = input.Width(); + } + } + + return true; +} + template bool loopyCachedImageExtract(aliceVision::image::Image & output, CachedImage & input, const BoundingBox & extractedInputBb) { diff --git a/src/aliceVision/panorama/laplacianCompositer.hpp b/src/aliceVision/panorama/laplacianCompositer.hpp index b819f99dcc..c463ea5734 100644 --- a/src/aliceVision/panorama/laplacianCompositer.hpp +++ b/src/aliceVision/panorama/laplacianCompositer.hpp @@ -22,8 +22,16 @@ class LaplacianCompositer : public Compositer } - virtual bool initialize() + virtual bool initialize(const BoundingBox & outputRoi) { + _outputRoi = outputRoi; + + + if (_outputRoi.left < 0) return false; + if (_outputRoi.top < 0) return false; + if (_outputRoi.getRight() >= _panoramaWidth) return false; + if (_outputRoi.getBottom() >= _panoramaHeight) return false; + return _pyramidPanorama.initialize(); } @@ -32,9 +40,9 @@ class LaplacianCompositer : public Compositer return _gaussianFilterRadius; } - virtual bool append(aliceVision::image::Image& color, - aliceVision::image::Image& inputMask, - aliceVision::image::Image& inputWeights, + virtual bool append(const aliceVision::image::Image& color, + const aliceVision::image::Image& inputMask, + const aliceVision::image::Image& inputWeights, int offsetX, int offsetY) { // Make sure input is compatible with pyramid processing @@ -44,17 +52,12 @@ class LaplacianCompositer : public Compositer aliceVision::image::Image maskPot; aliceVision::image::Image weightsPot; - + + makeImagePyramidCompatible(colorPot, newOffsetX, newOffsetY, color, offsetX, offsetY, getBorderSize(), _bands); - color = aliceVision::image::Image(); - makeImagePyramidCompatible(maskPot, newOffsetX, newOffsetY, inputMask, offsetX, offsetY, getBorderSize(), _bands); - inputMask = aliceVision::image::Image(); - makeImagePyramidCompatible(weightsPot, newOffsetX, newOffsetY, inputWeights, offsetX, offsetY, getBorderSize(), _bands); - inputWeights = aliceVision::image::Image(); - - + // Fill Color images masked parts with fake but coherent info aliceVision::image::Image feathered; @@ -101,16 +104,16 @@ class LaplacianCompositer : public Compositer virtual bool terminate() { - _panorama = image::Image(_panoramaWidth, _panoramaHeight, true, image::RGBAfColor(0.0f, 0.0f, 0.0f, 0.0f)); + _panorama = image::Image(_outputRoi.width, _outputRoi.height, true, image::RGBAfColor(0.0f, 0.0f, 0.0f, 0.0f)); - if (!_pyramidPanorama.rebuild(_panorama)) + if (!_pyramidPanorama.rebuild(_panorama, _outputRoi)) { return false; } - for (int i = 0; i < _panorama.Height(); i++) + for (int i = 0; i < _outputRoi.height; i++) { - for (int j = 0; j < _panorama.Width(); j++) + for (int j = 0; j < _outputRoi.width; j++) { image::RGBAfColor c = _panorama(i, j); diff --git a/src/aliceVision/panorama/laplacianPyramid.cpp b/src/aliceVision/panorama/laplacianPyramid.cpp index c2e811e042..5055cb7e22 100644 --- a/src/aliceVision/panorama/laplacianPyramid.cpp +++ b/src/aliceVision/panorama/laplacianPyramid.cpp @@ -104,8 +104,6 @@ bool LaplacianPyramid::apply(const aliceVision::image::Image& for(int l = initialLevel; l < _levels.size() - 1; l++) { - image::Image & img = _levels[l]; - BoundingBox inputBbox; inputBbox.left = offsetX; inputBbox.top = offsetY; @@ -287,7 +285,7 @@ bool LaplacianPyramid::merge(const aliceVision::image::Image& return true; } -bool LaplacianPyramid::rebuild(image::Image& output) +bool LaplacianPyramid::rebuild(image::Image& output, const BoundingBox & roi) { for (InputInfo & iinfo : _inputInfos) { @@ -365,15 +363,19 @@ bool LaplacianPyramid::rebuild(image::Image& output) image::Image & level = _levels[0]; image::Image & weight = _weights[0]; - for(int i = 0; i < output.Height(); i++) + for(int i = 0; i < roi.height; i++) { - for(int j = 0; j < output.Width(); j++) + int y = i + roi.top; + + for(int j = 0; j < roi.width; j++) { - output(i, j).r() = level(i, j).r(); - output(i, j).g() = level(i, j).g(); - output(i, j).b() = level(i, j).b(); + int x = j + roi.left; + + output(i, j).r() = level(y, x).r(); + output(i, j).g() = level(y, x).g(); + output(i, j).b() = level(y, x).b(); - if (weight(i, j) < 1e-6) + if (weight(y, x) < 1e-6) { output(i, j).a() = 0.0f; } diff --git a/src/aliceVision/panorama/laplacianPyramid.hpp b/src/aliceVision/panorama/laplacianPyramid.hpp index a167d1522d..926435ebcc 100644 --- a/src/aliceVision/panorama/laplacianPyramid.hpp +++ b/src/aliceVision/panorama/laplacianPyramid.hpp @@ -35,7 +35,7 @@ class LaplacianPyramid const aliceVision::image::Image& oweight, size_t level, int offset_x, int offset_y); - bool rebuild(image::Image& output); + bool rebuild(image::Image& output, const BoundingBox & roi); private: int _baseWidth; diff --git a/src/aliceVision/panorama/panoramaMap.cpp b/src/aliceVision/panorama/panoramaMap.cpp index e88e165428..24724d0231 100644 --- a/src/aliceVision/panorama/panoramaMap.cpp +++ b/src/aliceVision/panorama/panoramaMap.cpp @@ -24,6 +24,7 @@ bool PanoramaMap::append(IndexT index, const BoundingBox & box) bool PanoramaMap::intersect(const BoundingBox & box1, const BoundingBox & box2) const { + BoundingBox extentedBox1 = box1.divide(_scale).dilate(_borderSize).multiply(_scale); BoundingBox extentedBox2 = box2.divide(_scale).dilate(_borderSize).multiply(_scale); @@ -90,11 +91,14 @@ bool PanoramaMap::getIntersectionsList(std::vector & intersections, BoundingBox intersectionSmall = referenceBoundingBoxReduced.intersectionWith(otherBoundingBoxReduced); if (!intersectionSmall.isEmpty()) { - currentBoundingBoxes.push_back(otherBoundingBox); - BoundingBox intersection = intersectionSmall.multiply(_scale); intersection = intersection.limitInside(otherBoundingBox); - intersections.push_back(intersection); + + if (!intersection.isEmpty()) + { + intersections.push_back(intersection); + currentBoundingBoxes.push_back(otherBoundingBox); + } } } @@ -104,13 +108,17 @@ bool PanoramaMap::getIntersectionsList(std::vector & intersections, otherBoundingBoxLoop.left -= _panoramaWidth; BoundingBox otherBoundingBoxReduced = otherBoundingBoxLoop.divide(_scale).dilate(_borderSize); BoundingBox intersectionSmall = referenceBoundingBoxReduced.intersectionWith(otherBoundingBoxReduced); + if (!intersectionSmall.isEmpty()) - { - currentBoundingBoxes.push_back(otherBoundingBoxLoop); - + { BoundingBox intersection = intersectionSmall.multiply(_scale); intersection = intersection.limitInside(otherBoundingBoxLoop); - intersections.push_back(intersection); + + if (!intersection.isEmpty()) + { + intersections.push_back(intersection); + currentBoundingBoxes.push_back(otherBoundingBoxLoop); + } } } @@ -121,12 +129,34 @@ bool PanoramaMap::getIntersectionsList(std::vector & intersections, BoundingBox otherBoundingBoxReduced = otherBoundingBoxLoop.divide(_scale).dilate(_borderSize); BoundingBox intersectionSmall = referenceBoundingBoxReduced.intersectionWith(otherBoundingBoxReduced); if (!intersectionSmall.isEmpty()) - { - currentBoundingBoxes.push_back(otherBoundingBoxLoop); - + { BoundingBox intersection = intersectionSmall.multiply(_scale); intersection = intersection.limitInside(otherBoundingBoxLoop); - intersections.push_back(intersection); + + if (!intersection.isEmpty()) + { + intersections.push_back(intersection); + currentBoundingBoxes.push_back(otherBoundingBoxLoop); + } + } + } + + // Shift to check loop + { + BoundingBox otherBoundingBoxLoop = otherBoundingBox; + otherBoundingBoxLoop.left += _panoramaWidth + _panoramaWidth; + BoundingBox otherBoundingBoxReduced = otherBoundingBoxLoop.divide(_scale).dilate(_borderSize); + BoundingBox intersectionSmall = referenceBoundingBoxReduced.intersectionWith(otherBoundingBoxReduced); + if (!intersectionSmall.isEmpty()) + { + BoundingBox intersection = intersectionSmall.multiply(_scale); + intersection = intersection.limitInside(otherBoundingBoxLoop); + + if (!intersection.isEmpty()) + { + intersections.push_back(intersection); + currentBoundingBoxes.push_back(otherBoundingBoxLoop); + } } } diff --git a/src/aliceVision/panorama/panoramaMap.hpp b/src/aliceVision/panorama/panoramaMap.hpp index a033ad28da..2ff2685124 100644 --- a/src/aliceVision/panorama/panoramaMap.hpp +++ b/src/aliceVision/panorama/panoramaMap.hpp @@ -10,6 +10,7 @@ namespace aliceVision { + class PanoramaMap { public: @@ -25,12 +26,12 @@ class PanoramaMap bool getOverlaps(std::list & overlaps, IndexT reference) const; - size_t getWidth() const + int getWidth() const { return _panoramaWidth; } - size_t getHeight() const + int getHeight() const { return _panoramaHeight; } @@ -40,6 +41,11 @@ class PanoramaMap return _scale; } + size_t getBorderSize() const + { + return _borderSize; + } + bool getBoundingBox(BoundingBox & bb, const IndexT & id) const { if (_map.find(id) == _map.end()) diff --git a/src/software/pipeline/main_panoramaCompositing.cpp b/src/software/pipeline/main_panoramaCompositing.cpp index a4f8a552ff..10a85e90cc 100644 --- a/src/software/pipeline/main_panoramaCompositing.cpp +++ b/src/software/pipeline/main_panoramaCompositing.cpp @@ -61,7 +61,7 @@ size_t getCompositingOptimalScale(int width, int height) size_t optimal_scale = size_t(floor(std::log2(double(minsize) / gaussianFilterSize))); - return (optimal_scale - 1); + return (optimal_scale); } std::unique_ptr buildMap(const sfmData::SfMData & sfmData, const std::string & inputPath, const size_t borderSize) @@ -95,6 +95,8 @@ std::unique_ptr buildMap(const sfmData::SfMData & sfmData, const st bb.width = width; bb.height = height; + if (viewIt.first == 0) continue; + listBoundingBox.push_back(std::make_pair(viewIt.first, bb)); size_t scale = getCompositingOptimalScale(width, height); if (scale < min_scale) @@ -103,7 +105,6 @@ std::unique_ptr buildMap(const sfmData::SfMData & sfmData, const st } } - std::unique_ptr ret(new PanoramaMap(panoramaSize.first, panoramaSize.second, min_scale, borderSize)); for (const auto & bbitem : listBoundingBox) { @@ -113,82 +114,22 @@ std::unique_ptr buildMap(const sfmData::SfMData & sfmData, const st return ret; } -bool computeWTALabels(image::Image & labels, std::list & views, const std::string & inputPath, const PanoramaMap & map, const IndexT & referenceIndex) -{ - ALICEVISION_LOG_INFO("Estimating initial labels for panorama"); - - BoundingBox referenceBoundingBox; - - if (!map.getBoundingBox(referenceBoundingBox, referenceIndex)) - { - return false; - } - - WTASeams seams(referenceBoundingBox.width, referenceBoundingBox.height); - - for (const IndexT & currentId : views) - { - // Load mask - const std::string maskPath = (fs::path(inputPath) / (std::to_string(currentId) + "_mask.exr")).string(); - ALICEVISION_LOG_TRACE("Load mask with path " << maskPath); - image::Image mask; - image::readImage(maskPath, mask, image::EImageColorSpace::NO_CONVERSION); - - // Get offset - oiio::ParamValueList metadata = image::readImageMetadata(maskPath); - const int offsetX = metadata.find("AliceVision:offsetX")->get_int(); - const int offsetY = metadata.find("AliceVision:offsetY")->get_int(); - - // Load Weights - const std::string weightsPath = (fs::path(inputPath) / (std::to_string(currentId) + "_weight.exr")).string(); - ALICEVISION_LOG_TRACE("Load weights with path " << weightsPath); - image::Image weights; - image::readImage(weightsPath, weights, image::EImageColorSpace::NO_CONVERSION); - - - std::vector intersections; - std::vector currentBoundingBoxes; - if (!map.getIntersectionsList(intersections, currentBoundingBoxes, referenceIndex, currentId)) - { - continue; - } - - for (const BoundingBox & bbox : currentBoundingBoxes) - { - if (!seams.append(mask, weights, currentId, bbox.left - referenceBoundingBox.left, bbox.top - referenceBoundingBox.top)) - { - return false; - } - } - } - - labels = seams.getLabels(); - - return true; -} - -bool processImage(const PanoramaMap & panoramaMap, const std::string & compositerType, const std::string & warpingFolder, const std::string & outputFolder, const image::EStorageDataType & storageDataType, const IndexT & viewReference) +bool processImage(const PanoramaMap & panoramaMap, const std::string & compositerType, const std::string & warpingFolder, const std::string & labelsFilePath, const std::string & outputFolder, const image::EStorageDataType & storageDataType, const IndexT & viewReference) { - // Get the list of input which should be processed for this reference view bounding box - std::list overlaps; - if (!panoramaMap.getOverlaps(overlaps, viewReference)) - { - ALICEVISION_LOG_ERROR("Problem analyzing neighboorhood"); - return EXIT_FAILURE; - } - - // Add the current input also for simpler processing - overlaps.push_back(viewReference); - // Get the input bounding box to define the ROI BoundingBox referenceBoundingBox; if (!panoramaMap.getBoundingBox(referenceBoundingBox, viewReference)) { ALICEVISION_LOG_ERROR("Problem getting reference bounding box"); - return EXIT_FAILURE; + return false; } - //Create a compositer depending on what we need + // The laplacian pyramid must also contains some pixels outside of the bounding box to make sure + // there is a continuity between all the "views" of the panorama. + BoundingBox panoramaBoundingBox = referenceBoundingBox; + + + //Create a compositer depending on what was requested bool needWeights; bool needSeams; std::unique_ptr compositer; @@ -196,7 +137,10 @@ bool processImage(const PanoramaMap & panoramaMap, const std::string & composite { needWeights = false; needSeams = true; - compositer = std::unique_ptr(new LaplacianCompositer(referenceBoundingBox.width, referenceBoundingBox.height, panoramaMap.getScale())); + + //Enlarge the panorama boundingbox to allow consider neighboor pixels even at small scale + panoramaBoundingBox = referenceBoundingBox.divide(panoramaMap.getScale()).dilate(panoramaMap.getBorderSize()).multiply(panoramaMap.getScale()); + compositer = std::unique_ptr(new LaplacianCompositer(panoramaBoundingBox.width, panoramaBoundingBox.height, panoramaMap.getScale())); } else if (compositerType == "alpha") { @@ -210,32 +154,219 @@ bool processImage(const PanoramaMap & panoramaMap, const std::string & composite needSeams = false; compositer = std::unique_ptr(new Compositer(referenceBoundingBox.width, referenceBoundingBox.height)); } - - // Compositer initialization - if (!compositer->initialize()) + + + // Get the list of input which should be processed for this reference view bounding box + std::list overlappingViews; + if (!panoramaMap.getOverlaps(overlappingViews, viewReference)) { - ALICEVISION_LOG_ERROR("Error initializing panorama"); + ALICEVISION_LOG_ERROR("Problem analyzing neighboorhood"); return false; } + // Add the reference input for simpler processing + overlappingViews.push_back(viewReference); + + + // Compute the bounding box of the intersections with the reference bounding box + // (which may be larger than the reference Bounding box because of dilatation) + BoundingBox globalUnionBoundingBox; + for (IndexT viewCurrent : overlappingViews) + { + //Compute list of intersection between this view and the reference view + std::vector intersections; + std::vector currentBoundingBoxes; + if (!panoramaMap.getIntersectionsList(intersections, currentBoundingBoxes, viewReference, viewCurrent)) + { + continue; + } + + for (BoundingBox & bb : intersections) + { + globalUnionBoundingBox = globalUnionBoundingBox.unionWith(bb); + } + } + + // Building a map of visible pixels + image::Image> visiblePixels(globalUnionBoundingBox.width, globalUnionBoundingBox.height, true); + for (IndexT viewCurrent : overlappingViews) + { + // Load mask + const std::string maskPath = (fs::path(warpingFolder) / (std::to_string(viewCurrent) + "_mask.exr")).string(); + ALICEVISION_LOG_TRACE("Load mask with path " << maskPath); + image::Image mask; + image::readImage(maskPath, mask, image::EImageColorSpace::NO_CONVERSION); + + // Compute list of intersection between this view and the reference view + std::vector intersections; + std::vector currentBoundingBoxes; + if (!panoramaMap.getIntersectionsList(intersections, currentBoundingBoxes, viewReference, viewCurrent)) + { + continue; + } + + for (int indexIntersection = 0; indexIntersection < intersections.size(); indexIntersection++) + { + const BoundingBox & bbox = currentBoundingBoxes[indexIntersection]; + const BoundingBox & bboxIntersect = intersections[indexIntersection]; + + for (int i = 0; i < mask.Height(); i++) + { + int y = bbox.top + i - globalUnionBoundingBox.top; + if (y < 0 || y >= globalUnionBoundingBox.height) + { + continue; + } + + for (int j = 0; j < mask.Width(); j++) + { + if (!mask(i, j)) + { + continue; + } + + int x = bbox.left + j - globalUnionBoundingBox.left; + if (x < 0 || x >= globalUnionBoundingBox.width) + { + continue; + } + + + visiblePixels(y, x).push_back(viewCurrent); + } + } + } + } + + // Compute initial seams - image::Image labels; + image::Image referenceLabels; if (needSeams) { - if (!computeWTALabels(labels, overlaps, warpingFolder, panoramaMap, viewReference)) + image::Image panoramaLabels; + image::readImage(labelsFilePath, panoramaLabels, image::EImageColorSpace::NO_CONVERSION); + + double scaleX = double(panoramaLabels.Width()) / double(panoramaMap.getWidth()); + double scaleY = double(panoramaLabels.Height()) / double(panoramaMap.getHeight()); + + referenceLabels = image::Image(globalUnionBoundingBox.width, globalUnionBoundingBox.height, true, UndefinedIndexT); + + for (int i = 0; i < globalUnionBoundingBox.height; i++) { - ALICEVISION_LOG_ERROR("Error estimating panorama labels for this input"); - return false; + int y = i + globalUnionBoundingBox.top; + int scaledY = int(floor(scaleY * double(y))); + + for (int j = 0; j < globalUnionBoundingBox.width; j++) + { + int x = j + globalUnionBoundingBox.left; + int scaledX = int(floor(scaleX * double(x))); + + if (scaledX < 0) + { + scaledX += panoramaLabels.Width(); + } + + if (scaledX >= panoramaLabels.Width()) + { + scaledX -= panoramaLabels.Width(); + } + + if (scaledX < 0) continue; + if (scaledX >= panoramaLabels.Width()) continue; + + IndexT label = panoramaLabels(scaledY, scaledX); + + bool found = false; + auto & listValid = visiblePixels(i, j); + for (auto item : listValid) + { + if (item == label) + { + found = true; + break; + } + } + + if (found) + { + referenceLabels(i, j) = label; + continue; + } + + found = false; + for (int k = -1; k <= 1; k++) + { + int nscaledY = scaledY + k; + if (nscaledY < 0) continue; + if (nscaledY >= panoramaLabels.Height()) continue; + + for (int l = -1; l <= 1; l++) + { + if (k == 0 && l == 0) continue; + + int nscaledX = scaledX + l; + if (nscaledX < 0) continue; + if (nscaledX >= panoramaLabels.Width()) continue; + + IndexT otherlabel = panoramaLabels(nscaledY, nscaledX); + for (auto item : listValid) + { + if (item == otherlabel) + { + label = otherlabel; + found = true; + break; + } + } + + if (found) break; + } + + if (found) break; + } + + if (!found) + { + std::cout << "oups"<< std::endl; + referenceLabels(i, j) = UndefinedIndexT; + continue; + } + + referenceLabels(i, j) = label; + } } + } + + // Compute the roi of the output inside the compositer computed + // image (which may be larger than required for algorithmic reasons) + BoundingBox bbRoi; + bbRoi.left = referenceBoundingBox.left - panoramaBoundingBox.left; + bbRoi.top = referenceBoundingBox.top - panoramaBoundingBox.top; + bbRoi.width = referenceBoundingBox.width; + bbRoi.height = referenceBoundingBox.height; + + // Compositer initialization + if (!compositer->initialize(bbRoi)) + { + ALICEVISION_LOG_ERROR("Error initializing panorama"); + return false; } int posCurrent = 0; - for (IndexT viewCurrent : overlaps) + for (IndexT viewCurrent : overlappingViews) { posCurrent++; - ALICEVISION_LOG_INFO("Processing input " << posCurrent << "/" << overlaps.size()); + ALICEVISION_LOG_INFO("Processing input " << posCurrent << "/" << overlappingViews.size()); + + // Compute list of intersection between this view and the reference view + std::vector intersections; + std::vector currentBoundingBoxes; + if (!panoramaMap.getIntersectionsList(intersections, currentBoundingBoxes, viewReference, viewCurrent)) + { + continue; + } - // Load image and convert it to linear colorspace + // Load image const std::string imagePath = (fs::path(warpingFolder) / (std::to_string(viewCurrent) + ".exr")).string(); ALICEVISION_LOG_TRACE("Load image with path " << imagePath); image::Image source; @@ -247,12 +378,13 @@ bool processImage(const PanoramaMap & panoramaMap, const std::string & composite image::Image mask; image::readImage(maskPath, mask, image::EImageColorSpace::NO_CONVERSION); - //Compute list of intersection between this view and the reference view - std::vector intersections; - std::vector currentBoundingBoxes; - if (!panoramaMap.getIntersectionsList(intersections, currentBoundingBoxes, viewReference, viewCurrent)) + // Load weights image if needed + image::Image weights; + if (needWeights) { - continue; + const std::string weightsPath = (fs::path(warpingFolder) / (std::to_string(viewCurrent) + "_weight.exr")).string(); + ALICEVISION_LOG_TRACE("Load weights with path " << weightsPath); + image::readImage(weightsPath, weights, image::EImageColorSpace::NO_CONVERSION); } for (int indexIntersection = 0; indexIntersection < intersections.size(); indexIntersection++) @@ -260,6 +392,20 @@ bool processImage(const PanoramaMap & panoramaMap, const std::string & composite const BoundingBox & bbox = currentBoundingBoxes[indexIntersection]; const BoundingBox & bboxIntersect = intersections[indexIntersection]; + + if (needSeams) + { + int left = bboxIntersect.left - globalUnionBoundingBox.left; + int top = bboxIntersect.top - globalUnionBoundingBox.top; + + weights = image::Image(bboxIntersect.width, bboxIntersect.height); + if (!getMaskFromLabels(weights, referenceLabels, viewCurrent, left, top)) + { + ALICEVISION_LOG_ERROR("Error estimating seams image"); + return false; + } + } + BoundingBox cutBoundingBox; cutBoundingBox.left = bboxIntersect.left - bbox.left; cutBoundingBox.top = bboxIntersect.top - bbox.top; @@ -270,40 +416,16 @@ bool processImage(const PanoramaMap & panoramaMap, const std::string & composite continue; } - - image::Image weights; - if (needWeights) - { - const std::string weightsPath = (fs::path(warpingFolder) / (std::to_string(viewCurrent) + "_weight.exr")).string(); - ALICEVISION_LOG_TRACE("Load weights with path " << weightsPath); - image::readImage(weightsPath, weights, image::EImageColorSpace::NO_CONVERSION); - } - else if (needSeams) - { - weights = image::Image(mask.Width(), mask.Height()); - if (!getMaskFromLabels(weights, labels, viewCurrent, bbox.left - referenceBoundingBox.left, bbox.top - referenceBoundingBox.top)) - { - ALICEVISION_LOG_ERROR("Error estimating seams image"); - return EXIT_FAILURE; - } - } - image::Image subsource(cutBoundingBox.width, cutBoundingBox.height); - image::Image submask(cutBoundingBox.width, cutBoundingBox.height); - image::Image subweights(cutBoundingBox.width, cutBoundingBox.height); + image::Image submask(cutBoundingBox.width, cutBoundingBox.height); subsource = source.block(cutBoundingBox.top, cutBoundingBox.left, cutBoundingBox.height, cutBoundingBox.width); - submask = mask.block(cutBoundingBox.top, cutBoundingBox.left, cutBoundingBox.height, cutBoundingBox.width); + submask = mask.block(cutBoundingBox.top, cutBoundingBox.left, cutBoundingBox.height, cutBoundingBox.width); - if (weights.Width() > 0) - { - subweights = weights.block(cutBoundingBox.top, cutBoundingBox.left, cutBoundingBox.height, cutBoundingBox.width); - } - - if (!compositer->append(subsource, submask, subweights, bboxIntersect.left - referenceBoundingBox.left, bboxIntersect.top - referenceBoundingBox.top)) + if (!compositer->append(subsource, submask, weights, referenceBoundingBox.left - panoramaBoundingBox.left + bboxIntersect.left - referenceBoundingBox.left , referenceBoundingBox.top - panoramaBoundingBox.top + bboxIntersect.top - referenceBoundingBox.top)) { ALICEVISION_LOG_INFO("Error in compositer append"); - return EXIT_FAILURE; + return false; } } } @@ -313,11 +435,12 @@ bool processImage(const PanoramaMap & panoramaMap, const std::string & composite if (!compositer->terminate()) { ALICEVISION_LOG_ERROR("Error terminating panorama"); + return false; } const std::string viewIdStr = std::to_string(viewReference); const std::string outputFilePath = (fs::path(outputFolder) / (viewIdStr + ".exr")).string(); - image::Image output = compositer->getOutput(); + image::Image & output = compositer->getOutput(); if (storageDataType == image::EStorageDataType::HalfFinite) { @@ -349,12 +472,13 @@ bool processImage(const PanoramaMap & panoramaMap, const std::string & composite image::writeImage(outputFilePath, output, image::EImageColorSpace::LINEAR, metadata); - return EXIT_SUCCESS; + return true; } int aliceVision_main(int argc, char** argv) { std::string sfmDataFilepath; + std::string labelsFilepath; std::string warpingFolder; std::string outputFolder; std::string compositerType = "multiband"; @@ -386,7 +510,8 @@ int aliceVision_main(int argc, char** argv) ("storageDataType", po::value(&storageDataType)->default_value(storageDataType), ("Storage data type: " + image::EStorageDataType_informations()).c_str()) ("rangeIteration", po::value(&rangeIteration)->default_value(rangeIteration), "Range chunk id.") ("rangeSize", po::value(&rangeSize)->default_value(rangeSize), "Range size.") - ("maxThreads", po::value(&maxThreads)->default_value(maxThreads), "max number of threads to use."); + ("maxThreads", po::value(&maxThreads)->default_value(maxThreads), "max number of threads to use.") + ("labels,l", po::value(&labelsFilepath)->required(), "Labels image from seams estimation."); allParams.add(optionalParams); // Setup log level given command line @@ -490,21 +615,21 @@ int aliceVision_main(int argc, char** argv) return EXIT_FAILURE; } - const std::vector & chunk = chunks[rangeIteration]; bool succeeded = true; omp_set_num_threads(std::min(omp_get_thread_limit(), std::max(0, maxThreads))); - #pragma omp parallel for + //#pragma omp parallel for for (std::size_t posReference = 0; posReference < chunk.size(); posReference++) { ALICEVISION_LOG_INFO("processing input region " << posReference + 1 << "/" << chunk.size()); IndexT viewReference = chunk[posReference]; - - if (!processImage(*panoramaMap, compositerType, warpingFolder, outputFolder, storageDataType, viewReference)) + if (viewReference == 0) continue; + + if (!processImage(*panoramaMap, compositerType, warpingFolder, labelsFilepath, outputFolder, storageDataType, viewReference)) { succeeded = false; continue; From 72724877535c21b3b2b9f5ffc8a79aabe00dd28b Mon Sep 17 00:00:00 2001 From: Fabien Date: Mon, 11 Jan 2021 14:15:27 +0100 Subject: [PATCH 63/79] [image] removed erroneously compression for exr --- src/aliceVision/image/io.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aliceVision/image/io.cpp b/src/aliceVision/image/io.cpp index 58793c2786..a0a7ce7f29 100644 --- a/src/aliceVision/image/io.cpp +++ b/src/aliceVision/image/io.cpp @@ -444,7 +444,7 @@ void writeImage(const std::string& path, imageSpec.attribute("jpeg:subsampling", "4:4:4"); // if possible, always subsampling 4:4:4 for jpeg imageSpec.attribute("CompressionQuality", 100); // if possible, best compression quality - imageSpec.attribute("compression", isEXR ? "none" : "none"); // if possible, set compression (piz for EXR, none for the other) + imageSpec.attribute("compression", isEXR ? "piz" : "none"); // if possible, set compression (piz for EXR, none for the other) const oiio::ImageBuf imgBuf = oiio::ImageBuf(imageSpec, const_cast(image.data())); // original image buffer const oiio::ImageBuf* outBuf = &imgBuf; // buffer to write From ef2b91a87bb2ab94156bfe74b481f3bf85dcb9f0 Mon Sep 17 00:00:00 2001 From: Fabien Date: Tue, 12 Jan 2021 09:54:40 +0100 Subject: [PATCH 64/79] [image] add image io without float conversion --- src/aliceVision/image/io.cpp | 38 +++++++++++++++--------------------- 1 file changed, 16 insertions(+), 22 deletions(-) diff --git a/src/aliceVision/image/io.cpp b/src/aliceVision/image/io.cpp index a0a7ce7f29..66df90399f 100644 --- a/src/aliceVision/image/io.cpp +++ b/src/aliceVision/image/io.cpp @@ -356,35 +356,29 @@ void readImage(const std::string& path, } } -void readImageIndexT(const std::string& path, +template +void readImageNoFloat(const std::string& path, oiio::TypeDesc format, int nchannels, - Image& image, + Image& image, const ImageReadOptions & imageReadOptions) { // check requested channels number - assert(nchannels == 1 || nchannels >= 3); + assert(nchannels == 1); oiio::ImageSpec configSpec; oiio::ImageBuf inBuf(path, 0, 0, NULL, &configSpec); - inBuf.read(0, 0, true, oiio::TypeDesc::UINT32); // force image convertion to float (for grayscale and color space convertion) + inBuf.read(0, 0, true, format); if(!inBuf.initialized()) throw std::runtime_error("Cannot find/open image file '" + path + "'."); // check picture channels number - if(inBuf.spec().nchannels != 1 && inBuf.spec().nchannels < 3) + if(inBuf.spec().nchannels != 1) throw std::runtime_error("Can't load channels of image file '" + path + "'."); - // color conversion - if(imageReadOptions.outputColorSpace == EImageColorSpace::AUTO) - throw std::runtime_error("You must specify a requested color space for image file '" + path + "'."); - - const std::string& colorSpace = inBuf.spec().get_string_attribute("oiio:ColorSpace", "sRGB"); // default image color space is sRGB - ALICEVISION_LOG_TRACE("Read image " << path << " (encoded in " << colorSpace << " colorspace)."); - // copy pixels from oiio to eigen image.resize(inBuf.spec().width, inBuf.spec().height, false); @@ -497,10 +491,10 @@ void writeImage(const std::string& path, fs::rename(tmpPath, path); } -void writeImageIndexT(const std::string& path, +template +void writeImageNoFloat(const std::string& path, oiio::TypeDesc typeDesc, - int nchannels, - const Image& image, + const Image& image, EImageColorSpace imageColorSpace, const oiio::ParamValueList& metadata = oiio::ParamValueList()) { @@ -520,21 +514,21 @@ void writeImageIndexT(const std::string& path, imageColorSpace = EImageColorSpace::LINEAR; } - oiio::ImageSpec imageSpec(image.Width(), image.Height(), nchannels, typeDesc); + oiio::ImageSpec imageSpec(image.Width(), image.Height(), 1, typeDesc); imageSpec.extra_attribs = metadata; // add custom metadata imageSpec.attribute("jpeg:subsampling", "4:4:4"); // if possible, always subsampling 4:4:4 for jpeg imageSpec.attribute("CompressionQuality", 100); // if possible, best compression quality imageSpec.attribute("compression", isEXR ? "none" : "none"); // if possible, set compression (piz for EXR, none for the other) - const oiio::ImageBuf imgBuf = oiio::ImageBuf(imageSpec, const_cast(image.data())); // original image buffer + const oiio::ImageBuf imgBuf = oiio::ImageBuf(imageSpec, const_cast(image.data())); // original image buffer const oiio::ImageBuf* outBuf = &imgBuf; // buffer to write oiio::ImageBuf formatBuf; // buffer for image format modification if(isEXR) { - formatBuf.copy(*outBuf, oiio::TypeDesc::UINT32); // override format, use half instead of float + formatBuf.copy(*outBuf, typeDesc); // override format, use half instead of float outBuf = &formatBuf; } @@ -553,12 +547,12 @@ void readImage(const std::string& path, Image& image, const ImageReadOpti void readImage(const std::string& path, Image& image, const ImageReadOptions & imageReadOptions) { - readImage(path, oiio::TypeDesc::UINT8, 1, image, imageReadOptions); + readImageNoFloat(path, oiio::TypeDesc::UINT8, 1, image, imageReadOptions); } void readImage(const std::string& path, Image& image, const ImageReadOptions & imageReadOptions) { - readImageIndexT(path, oiio::TypeDesc::UINT32, 1, image, imageReadOptions); + readImageNoFloat(path, oiio::TypeDesc::UINT32, 1, image, imageReadOptions); } void readImage(const std::string& path, Image& image, const ImageReadOptions & imageReadOptions) @@ -583,7 +577,7 @@ void readImage(const std::string& path, Image& image, const ImageReadO void writeImage(const std::string& path, const Image& image, EImageColorSpace imageColorSpace, const oiio::ParamValueList& metadata) { - writeImage(path, oiio::TypeDesc::UINT8, 1, image, imageColorSpace, metadata); + writeImageNoFloat(path, oiio::TypeDesc::UINT8, image, imageColorSpace, metadata); } void writeImage(const std::string& path, const Image& image, EImageColorSpace imageColorSpace, const oiio::ParamValueList& metadata) @@ -593,7 +587,7 @@ void writeImage(const std::string& path, const Image& image, EImageColorSpa void writeImage(const std::string& path, const Image& image, EImageColorSpace imageColorSpace, const oiio::ParamValueList& metadata) { - writeImageIndexT(path, oiio::TypeDesc::UINT32, 1, image, imageColorSpace, metadata); + writeImageNoFloat(path, oiio::TypeDesc::UINT32, image, imageColorSpace, metadata); } void writeImage(const std::string& path, const Image& image, EImageColorSpace imageColorSpace, const oiio::ParamValueList& metadata) From 9d359ef8c8fa9c6369a95bda8d331bf0feb2e530 Mon Sep 17 00:00:00 2001 From: Fabien Date: Tue, 12 Jan 2021 12:28:23 +0100 Subject: [PATCH 65/79] [sfmData] add comments --- src/aliceVision/sfmData/View.cpp | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/aliceVision/sfmData/View.cpp b/src/aliceVision/sfmData/View.cpp index 76e31fbdb2..20aa62b9fa 100644 --- a/src/aliceVision/sfmData/View.cpp +++ b/src/aliceVision/sfmData/View.cpp @@ -59,6 +59,18 @@ float View::getCameraExposureSetting(const float referenceISO, const float refer float new_fnumber = fnumber * iso_2_aperture; float exp_increase = (new_fnumber / lReferenceFNumber) * (new_fnumber / lReferenceFNumber); + // If the aperture was more important for this image, this means that it received less light than with a default aperture + // This means also that if we want to simulate that all the image have the same aperture, we have to increase virtually th + // light received as if the aperture was smaller. So we increase the exposure time + + // If the iso is larger than the default value, this means that it recevied more light than with a default iso + // This means also that if we want to simulate that all the image have the same iso, we have to decrease virtually th + // light received as if the iso was smaller. So we decrease the exposure time or equivalent, increase the aperture value + + // Checks + // iso 20, f/2 = 2500 + // iso 40, f/2.8 = 2500 + return shutter * exp_increase; } From 4c26733f87b296cfa87a0ff7b095ed7b8f228293 Mon Sep 17 00:00:00 2001 From: Fabien Date: Tue, 12 Jan 2021 14:41:02 +0100 Subject: [PATCH 66/79] [image] make sure we can read 3 channels unisgned char images --- src/aliceVision/image/io.cpp | 29 ++++++++++--------- src/aliceVision/image/io.hpp | 8 +++++ .../pipeline/main_panoramaCompositing.cpp | 6 ++-- src/software/pipeline/main_panoramaSeams.cpp | 4 +-- 4 files changed, 29 insertions(+), 18 deletions(-) diff --git a/src/aliceVision/image/io.cpp b/src/aliceVision/image/io.cpp index 66df90399f..a24f91dd3e 100644 --- a/src/aliceVision/image/io.cpp +++ b/src/aliceVision/image/io.cpp @@ -359,13 +359,8 @@ void readImage(const std::string& path, template void readImageNoFloat(const std::string& path, oiio::TypeDesc format, - int nchannels, - Image& image, - const ImageReadOptions & imageReadOptions) + Image& image) { - // check requested channels number - assert(nchannels == 1); - oiio::ImageSpec configSpec; oiio::ImageBuf inBuf(path, 0, 0, NULL, &configSpec); @@ -373,19 +368,22 @@ void readImageNoFloat(const std::string& path, inBuf.read(0, 0, true, format); if(!inBuf.initialized()) + { throw std::runtime_error("Cannot find/open image file '" + path + "'."); + } // check picture channels number if(inBuf.spec().nchannels != 1) + { throw std::runtime_error("Can't load channels of image file '" + path + "'."); - - + } + // copy pixels from oiio to eigen image.resize(inBuf.spec().width, inBuf.spec().height, false); { oiio::ROI exportROI = inBuf.roi(); exportROI.chbegin = 0; - exportROI.chend = nchannels; + exportROI.chend = 1; inBuf.get_pixels(exportROI, format, image.data()); } @@ -547,12 +545,17 @@ void readImage(const std::string& path, Image& image, const ImageReadOpti void readImage(const std::string& path, Image& image, const ImageReadOptions & imageReadOptions) { - readImageNoFloat(path, oiio::TypeDesc::UINT8, 1, image, imageReadOptions); + readImage(path, oiio::TypeDesc::UINT8, 1, image, imageReadOptions); +} + +void readImageDirect(const std::string& path, Image& image) +{ + readImageNoFloat(path, oiio::TypeDesc::UINT8, image); } -void readImage(const std::string& path, Image& image, const ImageReadOptions & imageReadOptions) +void readImageDirect(const std::string& path, Image& image) { - readImageNoFloat(path, oiio::TypeDesc::UINT32, 1, image, imageReadOptions); + readImageNoFloat(path, oiio::TypeDesc::UINT32, image); } void readImage(const std::string& path, Image& image, const ImageReadOptions & imageReadOptions) @@ -582,7 +585,7 @@ void writeImage(const std::string& path, const Image& image, EIma void writeImage(const std::string& path, const Image& image, EImageColorSpace imageColorSpace, const oiio::ParamValueList& metadata) { - writeImage(path, oiio::TypeDesc::UINT32, 1, image, imageColorSpace, metadata); + writeImageNoFloat(path, oiio::TypeDesc::INT32, image, imageColorSpace, metadata); } void writeImage(const std::string& path, const Image& image, EImageColorSpace imageColorSpace, const oiio::ParamValueList& metadata) diff --git a/src/aliceVision/image/io.hpp b/src/aliceVision/image/io.hpp index fdc0133ff1..7906e239eb 100644 --- a/src/aliceVision/image/io.hpp +++ b/src/aliceVision/image/io.hpp @@ -188,6 +188,14 @@ void readImage(const std::string& path, Image& image, const ImageRead void readImage(const std::string& path, Image& image, const ImageReadOptions & imageReadOptions); void readImage(const std::string& path, Image& image, const ImageReadOptions & imageReadOptions); +/** + * @brief read an image with a given path and buffer without any processing such as color conversion + * @param[in] path The given path to the image + * @param[out] image The output image buffer + */ +void readImageDirect(const std::string& path, Image& image); +void readImageDirect(const std::string& path, Image& image); + /** * @brief write an image with a given path and buffer * @param[in] path The given path to the image diff --git a/src/software/pipeline/main_panoramaCompositing.cpp b/src/software/pipeline/main_panoramaCompositing.cpp index 10a85e90cc..844292f730 100644 --- a/src/software/pipeline/main_panoramaCompositing.cpp +++ b/src/software/pipeline/main_panoramaCompositing.cpp @@ -195,7 +195,7 @@ bool processImage(const PanoramaMap & panoramaMap, const std::string & composite const std::string maskPath = (fs::path(warpingFolder) / (std::to_string(viewCurrent) + "_mask.exr")).string(); ALICEVISION_LOG_TRACE("Load mask with path " << maskPath); image::Image mask; - image::readImage(maskPath, mask, image::EImageColorSpace::NO_CONVERSION); + image::readImageDirect(maskPath, mask); // Compute list of intersection between this view and the reference view std::vector intersections; @@ -244,7 +244,7 @@ bool processImage(const PanoramaMap & panoramaMap, const std::string & composite if (needSeams) { image::Image panoramaLabels; - image::readImage(labelsFilePath, panoramaLabels, image::EImageColorSpace::NO_CONVERSION); + image::readImageDirect(labelsFilePath, panoramaLabels); double scaleX = double(panoramaLabels.Width()) / double(panoramaMap.getWidth()); double scaleY = double(panoramaLabels.Height()) / double(panoramaMap.getHeight()); @@ -376,7 +376,7 @@ bool processImage(const PanoramaMap & panoramaMap, const std::string & composite const std::string maskPath = (fs::path(warpingFolder) / (std::to_string(viewCurrent) + "_mask.exr")).string(); ALICEVISION_LOG_TRACE("Load mask with path " << maskPath); image::Image mask; - image::readImage(maskPath, mask, image::EImageColorSpace::NO_CONVERSION); + image::readImageDirect(maskPath, mask); // Load weights image if needed image::Image weights; diff --git a/src/software/pipeline/main_panoramaSeams.cpp b/src/software/pipeline/main_panoramaSeams.cpp index fd0d622c0e..b1f297c06f 100644 --- a/src/software/pipeline/main_panoramaSeams.cpp +++ b/src/software/pipeline/main_panoramaSeams.cpp @@ -56,7 +56,7 @@ bool computeWTALabels(image::Image & labels, const std::vector mask; - image::readImage(maskPath, mask, image::EImageColorSpace::NO_CONVERSION); + image::readImageDirect(maskPath, mask); // Get offset oiio::ParamValueList metadata = image::readImageMetadata(maskPath); @@ -102,7 +102,7 @@ bool computeGCLabels(image::Image & labels, const std::vector mask; - image::readImage(maskPath, mask, image::EImageColorSpace::NO_CONVERSION); + image::readImageDirect(maskPath, mask); // Load Color const std::string colorsPath = (fs::path(inputPath) / (std::to_string(viewId) + ".exr")).string(); From b5ff757d5b8d67df477a204a25c9f720c3a2467d Mon Sep 17 00:00:00 2001 From: Fabien Date: Tue, 19 Jan 2021 09:59:47 +0100 Subject: [PATCH 67/79] [panorama] add debug features --- src/aliceVision/panorama/seams.cpp | 176 +++++++----------- src/aliceVision/panorama/seams.hpp | 5 +- .../pipeline/main_panoramaCompositing.cpp | 70 ++++++- 3 files changed, 134 insertions(+), 117 deletions(-) diff --git a/src/aliceVision/panorama/seams.cpp b/src/aliceVision/panorama/seams.cpp index b4330404a9..838320470c 100644 --- a/src/aliceVision/panorama/seams.cpp +++ b/src/aliceVision/panorama/seams.cpp @@ -87,87 +87,87 @@ bool computeSeamsMap(image::Image& seams, const image::Image& inout, const aliceVision::image::Image& mask, size_t offsetX, size_t offsetY) +void drawBorders(aliceVision::image::Image& inout, aliceVision::image::Image& mask, int offset_x, int offset_y) { - BoundingBox bb; - bb.left = offsetX; - bb.top = offsetY; - bb.width = mask.Width(); - bb.height = mask.Height(); - - const int border = 2; - BoundingBox dilatedBb = bb.dilate(border); - dilatedBb.clampTop(); - dilatedBb.clampBottom(inout.getHeight() - 1); - int leftBorder = bb.left - dilatedBb.left; - int topBorder = bb.top - dilatedBb.top; - - if (dilatedBb.left < 0) - { - dilatedBb.left += inout.getWidth(); - } - - image::Image extractedColor(dilatedBb.width, dilatedBb.height); - if (!loopyCachedImageExtract(extractedColor, inout, dilatedBb)) - { - return false; - } - - for(int i = 0; i < mask.Height(); i++) + for (int i = 0; i < mask.Height(); i++) { int j = 0; - int di = i + topBorder; - int dj = j + leftBorder; + int di = i + offset_y; + int dj = j + offset_x; + + if (di < 0 || dj < 0 || di >= inout.Height() || dj >= inout.Width()) + { + continue; + } if(mask(i, j)) { - extractedColor(di, dj) = image::RGBAfColor(0.0f, 1.0f, 0.0f, 1.0f); + inout(di, dj) = image::RGBAfColor(0.0f, 1.0f, 0.0f, 1.0f); } } for(int i = 0; i < mask.Height(); i++) { int j = mask.Width() - 1; - int di = i + topBorder; - int dj = j + leftBorder; + int di = i + offset_y; + int dj = j + offset_x; + + if (di < 0 || dj < 0 || di >= inout.Height() || dj >= inout.Width()) + { + continue; + } if(mask(i, j)) { - extractedColor(di, dj) = image::RGBAfColor(0.0f, 1.0f, 0.0f, 1.0f); + inout(di, dj) = image::RGBAfColor(0.0f, 1.0f, 0.0f, 1.0f); } } for(int j = 0; j < mask.Width(); j++) { int i = 0; - int di = i + topBorder; - int dj = j + leftBorder; - + int di = i + offset_y; + int dj = j + offset_x; + if (di < 0 || dj < 0 || di >= inout.Height() || dj >= inout.Width()) + { + continue; + } + if(mask(i, j)) { - extractedColor(di, dj) = image::RGBAfColor(0.0f, 1.0f, 0.0f, 1.0f); + inout(di, dj) = image::RGBAfColor(0.0f, 1.0f, 0.0f, 1.0f); } } for(int j = 0; j < mask.Width(); j++) { int i = mask.Height() - 1; - int di = i + topBorder; - int dj = j + leftBorder; + int di = i + offset_y; + int dj = j + offset_x; + if (di < 0 || dj < 0 || di >= inout.Height() || dj >= inout.Width()) + { + continue; + } if(mask(i, j)) { - extractedColor(di, dj) = image::RGBAfColor(0.0f, 1.0f, 0.0f, 1.0f); + inout(di, dj) = image::RGBAfColor(0.0f, 1.0f, 0.0f, 1.0f); } } for(int i = 1; i < mask.Height() - 1; i++) { - int di = i + topBorder; + + int di = i + offset_y; for(int j = 1; j < mask.Width() - 1; j++) { - int dj = j + leftBorder; + + int dj = j + offset_x; + if (di < 0 || dj < 0 || di >= inout.Height() || dj >= inout.Width()) + { + continue; + } if(!mask(i, j)) { @@ -181,93 +181,45 @@ bool drawBorders(CachedImage& inout, const aliceVision::image others &= mask(i, j + 1); others &= mask(i + 1, j - 1); others &= mask(i + 1, j + 1); - if(others) { + if(others) + { continue; } - extractedColor(di, dj) = image::RGBAfColor(0.0f, 1.0f, 0.0f, 1.0f); + inout(di, dj) = image::RGBAfColor(0.0f, 1.0f, 0.0f, 1.0f); } } - - BoundingBox inputBb = dilatedBb; - inputBb.left = 0; - inputBb.top = 0; - if (!loopyCachedImageAssign(inout, extractedColor, dilatedBb, inputBb)) - { - return false; - } - - return true; } -bool drawSeams(CachedImage& inout, CachedImage& labels) -{ - const int processingSize = 512; +void drawSeams(aliceVision::image::Image & inout, aliceVision::image::Image & labels, int offset_x, int offset_y) { - //Process image rectangle by rectangle - for (int y = 0; y < inout.getHeight(); y += processingSize) + for (int i = 1; i < labels.Height() - 1; i++) { - for (int x = 0; x < inout.getWidth(); x += processingSize) - { - //Compute the initial bouding box for this rectangle - BoundingBox currentBbox; - currentBbox.left = x; - currentBbox.top = y; - currentBbox.width = processingSize; - currentBbox.height = processingSize; - currentBbox.clampLeft(); - currentBbox.clampTop(); - currentBbox.clampRight(inout.getWidth() - 1); - currentBbox.clampBottom(inout.getHeight() - 1); - - image::Image extractedColor(currentBbox.width, currentBbox.height); - if (!loopyCachedImageExtract(extractedColor, inout, currentBbox)) - { - return false; - } + int di = i + offset_y; + if (di < 0 || di >= inout.Height()) continue; - image::Image extractedLabels(currentBbox.width, currentBbox.height); - if (!loopyCachedImageExtract(extractedLabels, labels, currentBbox)) - { - return false; - } - - for(int i = 1; i < extractedLabels.Height() - 1; i++) - { - - for(int j = 1; j < extractedLabels.Width() - 1; j++) - { - - IndexT label = extractedLabels(i, j); - IndexT same = true; + for (int j = 1; j < labels.Width() - 1; j++) + { + int dj = j + offset_x; + if (dj < 0 || dj >= inout.Width()) continue; - same &= (extractedLabels(i - 1, j - 1) == label); - same &= (extractedLabels(i - 1, j + 1) == label); - same &= (extractedLabels(i, j - 1) == label); - same &= (extractedLabels(i, j + 1) == label); - same &= (extractedLabels(i + 1, j - 1) == label); - same &= (extractedLabels(i + 1, j + 1) == label); + IndexT label = labels(i, j); + IndexT same = true; - if(same) - { - continue; - } + same &= (labels(i - 1, j - 1) == label); + same &= (labels(i - 1, j + 1) == label); + same &= (labels(i, j - 1) == label); + same &= (labels(i, j + 1) == label); + same &= (labels(i + 1, j - 1) == label); + same &= (labels(i + 1, j + 1) == label); - extractedColor(i, j) = image::RGBAfColor(1.0f, 0.0f, 0.0f, 1.0f); - } - } - - BoundingBox inputBbox = currentBbox; - inputBbox.left = 0; - inputBbox.top = 0; - if (!loopyCachedImageAssign(inout, extractedColor, currentBbox, inputBbox)) - { - return false; + if (same) { + continue; } + + inout(di, dj) = image::RGBAfColor(1.0f, 0.0f, 0.0f, 1.0f); } } - - return true; } bool WTASeams::append(const aliceVision::image::Image& inputMask, diff --git a/src/aliceVision/panorama/seams.hpp b/src/aliceVision/panorama/seams.hpp index 4af031747a..69d818e167 100644 --- a/src/aliceVision/panorama/seams.hpp +++ b/src/aliceVision/panorama/seams.hpp @@ -8,9 +8,10 @@ namespace aliceVision { -bool drawBorders(CachedImage & inout, const aliceVision::image::Image& mask, size_t offsetX, size_t offsetY); -bool drawSeams(CachedImage& inout, CachedImage& labels); +void drawBorders(aliceVision::image::Image & inout, aliceVision::image::Image & mask, int offset_x, int offset_y); + +void drawSeams(aliceVision::image::Image & inout, aliceVision::image::Image & labels, int offset_x, int offset_y); bool getMaskFromLabels(aliceVision::image::Image & mask, image::Image & labels, IndexT index, int offset_x, int offset_y); diff --git a/src/software/pipeline/main_panoramaCompositing.cpp b/src/software/pipeline/main_panoramaCompositing.cpp index 844292f730..acb7d788d5 100644 --- a/src/software/pipeline/main_panoramaCompositing.cpp +++ b/src/software/pipeline/main_panoramaCompositing.cpp @@ -114,7 +114,7 @@ std::unique_ptr buildMap(const sfmData::SfMData & sfmData, const st return ret; } -bool processImage(const PanoramaMap & panoramaMap, const std::string & compositerType, const std::string & warpingFolder, const std::string & labelsFilePath, const std::string & outputFolder, const image::EStorageDataType & storageDataType, const IndexT & viewReference) +bool processImage(const PanoramaMap & panoramaMap, const std::string & compositerType, const std::string & warpingFolder, const std::string & labelsFilePath, const std::string & outputFolder, const image::EStorageDataType & storageDataType, const IndexT & viewReference, bool showBorders, bool showSeams) { // Get the input bounding box to define the ROI BoundingBox referenceBoundingBox; @@ -187,6 +187,8 @@ bool processImage(const PanoramaMap & panoramaMap, const std::string & composite } } + ALICEVISION_LOG_INFO("Building the visibility map"); + // Building a map of visible pixels image::Image> visiblePixels(globalUnionBoundingBox.width, globalUnionBoundingBox.height, true); for (IndexT viewCurrent : overlappingViews) @@ -239,6 +241,8 @@ bool processImage(const PanoramaMap & panoramaMap, const std::string & composite } + ALICEVISION_LOG_INFO("Building the seams map"); + // Compute initial seams image::Image referenceLabels; if (needSeams) @@ -327,7 +331,6 @@ bool processImage(const PanoramaMap & panoramaMap, const std::string & composite if (!found) { - std::cout << "oups"<< std::endl; referenceLabels(i, j) = UndefinedIndexT; continue; } @@ -463,6 +466,54 @@ bool processImage(const PanoramaMap & panoramaMap, const std::string & composite } } + if (showBorders) + { + ALICEVISION_LOG_INFO("Draw borders"); + for (IndexT viewCurrent : overlappingViews) + { + // Compute list of intersection between this view and the reference view + std::vector intersections; + std::vector currentBoundingBoxes; + if (!panoramaMap.getIntersectionsList(intersections, currentBoundingBoxes, viewReference, viewCurrent)) + { + continue; + } + + // Load mask + const std::string maskPath = (fs::path(warpingFolder) / (std::to_string(viewCurrent) + "_mask.exr")).string(); + ALICEVISION_LOG_TRACE("Load mask with path " << maskPath); + image::Image mask; + image::readImageDirect(maskPath, mask); + + + for (int indexIntersection = 0; indexIntersection < intersections.size(); indexIntersection++) + { + const BoundingBox & bbox = currentBoundingBoxes[indexIntersection]; + const BoundingBox & bboxIntersect = intersections[indexIntersection]; + + BoundingBox cutBoundingBox; + cutBoundingBox.left = bboxIntersect.left - bbox.left; + cutBoundingBox.top = bboxIntersect.top - bbox.top; + cutBoundingBox.width = bboxIntersect.width; + cutBoundingBox.height = bboxIntersect.height; + if (cutBoundingBox.isEmpty()) + { + continue; + } + + image::Image submask(cutBoundingBox.width, cutBoundingBox.height); + submask = mask.block(cutBoundingBox.top, cutBoundingBox.left, cutBoundingBox.height, cutBoundingBox.width); + + drawBorders(output, submask, bboxIntersect.left - referenceBoundingBox.left, bboxIntersect.top - referenceBoundingBox.top); + } + } + } + + if (showSeams && needSeams) + { + drawSeams(output, referenceLabels, globalUnionBoundingBox.left - referenceBoundingBox.left, globalUnionBoundingBox.top- referenceBoundingBox.top); + } + oiio::ParamValueList metadata; metadata.push_back(oiio::ParamValue("AliceVision:storageDataType", EStorageDataType_enumToString(storageDataType))); metadata.push_back(oiio::ParamValue("AliceVision:offsetX", int(referenceBoundingBox.left))); @@ -482,9 +533,12 @@ int aliceVision_main(int argc, char** argv) std::string warpingFolder; std::string outputFolder; std::string compositerType = "multiband"; + std::string overlayType = "none"; int rangeIteration = -1; int rangeSize = 1; int maxThreads = 1; + bool showBorders = false; + bool showSeams = false; image::EStorageDataType storageDataType = image::EStorageDataType::Float; @@ -507,6 +561,7 @@ int aliceVision_main(int argc, char** argv) po::options_description optionalParams("Optional parameters"); optionalParams.add_options() ("compositerType,c", po::value(&compositerType)->required(), "Compositer Type [replace, alpha, multiband].") + ("overlayType,c", po::value(&overlayType)->required(), "Overlay Type [none, borders, seams, all].") ("storageDataType", po::value(&storageDataType)->default_value(storageDataType), ("Storage data type: " + image::EStorageDataType_informations()).c_str()) ("rangeIteration", po::value(&rangeIteration)->default_value(rangeIteration), "Range chunk id.") ("rangeSize", po::value(&rangeSize)->default_value(rangeSize), "Range size.") @@ -552,6 +607,15 @@ int aliceVision_main(int argc, char** argv) // Set verbose level given command line system::Logger::get()->setLogLevel(verboseLevel); + if (overlayType == "borders" || overlayType == "all") + { + showBorders = true; + } + + if (overlayType == "seams" || overlayType == "all") { + showSeams = true; + } + // load input scene sfmData::SfMData sfmData; if(!sfmDataIO::Load(sfmData, sfmDataFilepath, sfmDataIO::ESfMData(sfmDataIO::VIEWS | sfmDataIO::EXTRINSICS | sfmDataIO::INTRINSICS))) @@ -629,7 +693,7 @@ int aliceVision_main(int argc, char** argv) IndexT viewReference = chunk[posReference]; if (viewReference == 0) continue; - if (!processImage(*panoramaMap, compositerType, warpingFolder, labelsFilepath, outputFolder, storageDataType, viewReference)) + if (!processImage(*panoramaMap, compositerType, warpingFolder, labelsFilepath, outputFolder, storageDataType, viewReference, showBorders, showSeams)) { succeeded = false; continue; From 627f67b5c721a708308c43d84820d4d5523c6aee Mon Sep 17 00:00:00 2001 From: Fabien Date: Tue, 19 Jan 2021 11:26:41 +0100 Subject: [PATCH 68/79] [panorama] add other debug --- src/software/pipeline/main_panoramaCompositing.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/software/pipeline/main_panoramaCompositing.cpp b/src/software/pipeline/main_panoramaCompositing.cpp index acb7d788d5..a51b60e74e 100644 --- a/src/software/pipeline/main_panoramaCompositing.cpp +++ b/src/software/pipeline/main_panoramaCompositing.cpp @@ -390,6 +390,7 @@ bool processImage(const PanoramaMap & panoramaMap, const std::string & composite image::readImage(weightsPath, weights, image::EImageColorSpace::NO_CONVERSION); } + ALICEVISION_LOG_TRACE("Effective processing"); for (int indexIntersection = 0; indexIntersection < intersections.size(); indexIntersection++) { const BoundingBox & bbox = currentBoundingBoxes[indexIntersection]; From d3cbb583251343e327ee11f416df85dd78dfa2c9 Mon Sep 17 00:00:00 2001 From: Fabien Date: Wed, 20 Jan 2021 11:53:20 +0100 Subject: [PATCH 69/79] [panorama] memory optimization --- .../panorama/laplacianCompositer.hpp | 40 ++++----- src/aliceVision/panorama/laplacianPyramid.cpp | 88 +++++++------------ src/aliceVision/panorama/laplacianPyramid.hpp | 7 +- .../pipeline/main_panoramaCompositing.cpp | 6 ++ 4 files changed, 62 insertions(+), 79 deletions(-) diff --git a/src/aliceVision/panorama/laplacianCompositer.hpp b/src/aliceVision/panorama/laplacianCompositer.hpp index c463ea5734..85d443dbe4 100644 --- a/src/aliceVision/panorama/laplacianCompositer.hpp +++ b/src/aliceVision/panorama/laplacianCompositer.hpp @@ -45,23 +45,9 @@ class LaplacianCompositer : public Compositer const aliceVision::image::Image& inputWeights, int offsetX, int offsetY) { - // Make sure input is compatible with pyramid processing - // See comments inside function for details - int newOffsetX, newOffsetY; - aliceVision::image::Image colorPot; - aliceVision::image::Image maskPot; - aliceVision::image::Image weightsPot; - - - - makeImagePyramidCompatible(colorPot, newOffsetX, newOffsetY, color, offsetX, offsetY, getBorderSize(), _bands); - makeImagePyramidCompatible(maskPot, newOffsetX, newOffsetY, inputMask, offsetX, offsetY, getBorderSize(), _bands); - makeImagePyramidCompatible(weightsPot, newOffsetX, newOffsetY, inputWeights, offsetX, offsetY, getBorderSize(), _bands); - - // Fill Color images masked parts with fake but coherent info aliceVision::image::Image feathered; - if (!feathering(feathered, colorPot, maskPot)) + if (!feathering(feathered, color, inputMask)) { return false; } @@ -78,12 +64,12 @@ class LaplacianCompositer : public Compositer } // Convert mask to alpha layer - image::Image maskFloat(maskPot.Width(), maskPot.Height()); - for(int i = 0; i < maskPot.Height(); i++) + image::Image maskFloat(inputMask.Width(), inputMask.Height()); + for(int i = 0; i < inputMask.Height(); i++) { - for(int j = 0; j < maskPot.Width(); j++) + for(int j = 0; j < inputMask.Width(); j++) { - if(maskPot(i, j)) + if(inputMask(i, j)) { maskFloat(i, j) = 1.0f; } @@ -94,7 +80,21 @@ class LaplacianCompositer : public Compositer } } - if (!_pyramidPanorama.apply(feathered, maskFloat, weightsPot, 0, newOffsetX, newOffsetY)) + + BoundingBox bb; + bb.left = offsetX; + bb.top = offsetY; + bb.width = feathered.Width(); + bb.height = feathered.Height(); + + int scale = _bands - 1; + BoundingBox potbb = bb.divide(scale).dilate(getBorderSize()).multiply(scale); + + BoundingBox contentbb = bb; + contentbb.left = bb.left - potbb.left; + contentbb.top = bb.top - potbb.top; + + if (!_pyramidPanorama.apply(feathered, maskFloat, inputWeights, potbb, contentbb)) { return false; } diff --git a/src/aliceVision/panorama/laplacianPyramid.cpp b/src/aliceVision/panorama/laplacianPyramid.cpp index 5055cb7e22..2f8da47d1b 100644 --- a/src/aliceVision/panorama/laplacianPyramid.cpp +++ b/src/aliceVision/panorama/laplacianPyramid.cpp @@ -12,8 +12,13 @@ _baseWidth(base_width), _baseHeight(base_height), _maxLevels(max_levels) { + omp_init_lock(&_merge_lock); } +LaplacianPyramid::~LaplacianPyramid() +{ + omp_destroy_lock(&_merge_lock); +} bool LaplacianPyramid::initialize() { @@ -39,70 +44,35 @@ bool LaplacianPyramid::initialize() return true; } -bool LaplacianPyramid::augment(size_t newMaxLevels) -{ - ALICEVISION_LOG_INFO("augment number of levels to " << newMaxLevels); - - if(newMaxLevels <= _levels.size()) - { - return false; - } - - int oldMaxLevels = _levels.size(); - _maxLevels = newMaxLevels; - - - // Augment the number of levels in the pyramid - for (int level = oldMaxLevels; level < newMaxLevels; level++) - { - int width = _levels[_levels.size() - 1].Width(); - int height = _levels[_levels.size() - 1].Height(); - - int nextWidth = int(ceil(float(width) / 2.0f)); - int nextHeight = int(ceil(float(height) / 2.0f)); - - image::Image pyramidImage(nextWidth, nextHeight, true, image::RGBfColor(0.0f)); - image::Image pyramidWeights(nextWidth, nextHeight, true, 0.0f); - - _levels.push_back(pyramidImage); - _weights.push_back(pyramidWeights); - } - - std::vector localInfos = _inputInfos; - _inputInfos.clear(); - - for (InputInfo & iinfo : localInfos) - { - if (!apply(iinfo.color, iinfo.mask, iinfo.weights, oldMaxLevels - 1, iinfo.offsetX, iinfo.offsetY)) - { - return false; - } - } - - - return true; -} - - - bool LaplacianPyramid::apply(const aliceVision::image::Image& source, const aliceVision::image::Image& mask, const aliceVision::image::Image& weights, - size_t initialLevel, int offsetX, int offsetY) + const BoundingBox &outputBoundingBox, const BoundingBox &contentBoudingBox) { //We assume the input source has been feathered - //and resized to be a simili power of 2 for the needed scales. - int width = source.Width(); - int height = source.Height(); + //the outputBoundingBox has been scaled to accept the pyramid scales + int width = outputBoundingBox.width; + int height = outputBoundingBox.height; + int offsetX = outputBoundingBox.left; + int offsetY = outputBoundingBox.top; - image::Image currentColor = source; + image::Image currentColor(width, height, true, image::RGBfColor(0.0)); image::Image nextColor; - image::Image currentWeights = weights; + image::Image currentWeights(width, height, true, 0.0f); image::Image nextWeights; - image::Image currentMask = mask; + image::Image currentMask(width, height, true, 0.0f); image::Image nextMask; - for(int l = initialLevel; l < _levels.size() - 1; l++) + for (int i = 0; i < source.Height(); i++) + { + int di = contentBoudingBox.top + i; + + memcpy(¤tColor(di, contentBoudingBox.left), &source(i, 0), sizeof(image::RGBfColor) * source.Width()); + memcpy(¤tWeights(di, contentBoudingBox.left), &weights(i, 0), sizeof(float) * source.Width()); + memcpy(¤tMask(di, contentBoudingBox.left), &mask(i, 0), sizeof(float) * source.Width()); + } + + for(int l = 0; l < _levels.size() - 1; l++) { BoundingBox inputBbox; inputBbox.left = offsetX; @@ -133,7 +103,6 @@ bool LaplacianPyramid::apply(const aliceVision::image::Image& } } } - if (!convolveGaussian5x5(buf, bufMasked)) { @@ -227,7 +196,11 @@ bool LaplacianPyramid::apply(const aliceVision::image::Image& } //Merge this view with previous ones - if (!merge(currentColor, currentWeights, l, offsetX, offsetY)) + omp_set_lock(&_merge_lock); + bool res = merge(currentColor, currentWeights, l, offsetX, offsetY); + omp_unset_lock(&_merge_lock); + + if (!res) { return false; } @@ -251,7 +224,10 @@ bool LaplacianPyramid::apply(const aliceVision::image::Image& iinfo.color = currentColor; iinfo.mask = currentMask; iinfo.weights = currentWeights; + + omp_set_lock(&_merge_lock); _inputInfos.push_back(iinfo); + omp_unset_lock(&_merge_lock); return true; diff --git a/src/aliceVision/panorama/laplacianPyramid.hpp b/src/aliceVision/panorama/laplacianPyramid.hpp index 926435ebcc..1ad2956fa6 100644 --- a/src/aliceVision/panorama/laplacianPyramid.hpp +++ b/src/aliceVision/panorama/laplacianPyramid.hpp @@ -22,14 +22,14 @@ class LaplacianPyramid public: LaplacianPyramid(size_t base_width, size_t base_height, size_t max_levels); - bool initialize(); + virtual ~LaplacianPyramid(); - bool augment(size_t new_max_levels); + bool initialize(); bool apply(const aliceVision::image::Image& source, const aliceVision::image::Image& mask, const aliceVision::image::Image& weights, - size_t initialLevel, int offset_x, int offset_y); + const BoundingBox &outputBoundingBox, const BoundingBox &contentBoudingBox); bool merge(const aliceVision::image::Image& oimg, const aliceVision::image::Image& oweight, @@ -41,6 +41,7 @@ class LaplacianPyramid int _baseWidth; int _baseHeight; int _maxLevels; + omp_lock_t _merge_lock; std::vector> _levels; std::vector> _weights; diff --git a/src/software/pipeline/main_panoramaCompositing.cpp b/src/software/pipeline/main_panoramaCompositing.cpp index a51b60e74e..8b4ae749ff 100644 --- a/src/software/pipeline/main_panoramaCompositing.cpp +++ b/src/software/pipeline/main_panoramaCompositing.cpp @@ -369,6 +369,11 @@ bool processImage(const PanoramaMap & panoramaMap, const std::string & composite continue; } + if (intersections.size() == 0) + { + continue; + } + // Load image const std::string imagePath = (fs::path(warpingFolder) / (std::to_string(viewCurrent) + ".exr")).string(); ALICEVISION_LOG_TRACE("Load image with path " << imagePath); @@ -420,6 +425,7 @@ bool processImage(const PanoramaMap & panoramaMap, const std::string & composite continue; } + image::Image subsource(cutBoundingBox.width, cutBoundingBox.height); image::Image submask(cutBoundingBox.width, cutBoundingBox.height); From eb24b306d18ff1b2029b0b9bbf87056f6d47e29f Mon Sep 17 00:00:00 2001 From: Fabien Date: Wed, 20 Jan 2021 12:14:24 +0100 Subject: [PATCH 70/79] [panorama] some dirty hacks to reduce ram --- src/aliceVision/panorama/alphaCompositer.hpp | 6 +-- src/aliceVision/panorama/compositer.hpp | 6 +-- .../panorama/laplacianCompositer.hpp | 8 ++-- src/aliceVision/panorama/laplacianPyramid.cpp | 10 ++-- src/aliceVision/panorama/laplacianPyramid.hpp | 6 +-- .../pipeline/main_panoramaCompositing.cpp | 46 ++++++++++--------- 6 files changed, 45 insertions(+), 37 deletions(-) diff --git a/src/aliceVision/panorama/alphaCompositer.hpp b/src/aliceVision/panorama/alphaCompositer.hpp index ff2c5996b3..0d22c061ff 100644 --- a/src/aliceVision/panorama/alphaCompositer.hpp +++ b/src/aliceVision/panorama/alphaCompositer.hpp @@ -13,9 +13,9 @@ class AlphaCompositer : public Compositer { } - virtual bool append(const aliceVision::image::Image& color, - const aliceVision::image::Image& inputMask, - const aliceVision::image::Image& inputWeights, + virtual bool append(aliceVision::image::Image& color, + aliceVision::image::Image& inputMask, + aliceVision::image::Image& inputWeights, int offsetX, int offsetY) { offsetX -= _outputRoi.left; diff --git a/src/aliceVision/panorama/compositer.hpp b/src/aliceVision/panorama/compositer.hpp index fb3ff182b3..fd65844e29 100644 --- a/src/aliceVision/panorama/compositer.hpp +++ b/src/aliceVision/panorama/compositer.hpp @@ -24,9 +24,9 @@ class Compositer } - virtual bool append(const aliceVision::image::Image& color, - const aliceVision::image::Image& inputMask, - const aliceVision::image::Image& inputWeights, + virtual bool append(aliceVision::image::Image& color, + aliceVision::image::Image& inputMask, + aliceVision::image::Image& inputWeights, int offsetX, int offsetY) { offsetX -= _outputRoi.left; diff --git a/src/aliceVision/panorama/laplacianCompositer.hpp b/src/aliceVision/panorama/laplacianCompositer.hpp index 85d443dbe4..b12a629254 100644 --- a/src/aliceVision/panorama/laplacianCompositer.hpp +++ b/src/aliceVision/panorama/laplacianCompositer.hpp @@ -40,9 +40,9 @@ class LaplacianCompositer : public Compositer return _gaussianFilterRadius; } - virtual bool append(const aliceVision::image::Image& color, - const aliceVision::image::Image& inputMask, - const aliceVision::image::Image& inputWeights, + virtual bool append(aliceVision::image::Image& color, + aliceVision::image::Image& inputMask, + aliceVision::image::Image& inputWeights, int offsetX, int offsetY) { // Fill Color images masked parts with fake but coherent info @@ -52,6 +52,8 @@ class LaplacianCompositer : public Compositer return false; } + color = aliceVision::image::Image(); + // To log space for hdr for(int i = 0; i < feathered.Height(); i++) { diff --git a/src/aliceVision/panorama/laplacianPyramid.cpp b/src/aliceVision/panorama/laplacianPyramid.cpp index 2f8da47d1b..0d229de8c8 100644 --- a/src/aliceVision/panorama/laplacianPyramid.cpp +++ b/src/aliceVision/panorama/laplacianPyramid.cpp @@ -44,9 +44,9 @@ bool LaplacianPyramid::initialize() return true; } -bool LaplacianPyramid::apply(const aliceVision::image::Image& source, - const aliceVision::image::Image& mask, - const aliceVision::image::Image& weights, +bool LaplacianPyramid::apply(aliceVision::image::Image& source, + aliceVision::image::Image& mask, + aliceVision::image::Image& weights, const BoundingBox &outputBoundingBox, const BoundingBox &contentBoudingBox) { //We assume the input source has been feathered @@ -72,6 +72,10 @@ bool LaplacianPyramid::apply(const aliceVision::image::Image& memcpy(¤tMask(di, contentBoudingBox.left), &mask(i, 0), sizeof(float) * source.Width()); } + source = aliceVision::image::Image(); + mask = aliceVision::image::Image(); + weights = aliceVision::image::Image(); + for(int l = 0; l < _levels.size() - 1; l++) { BoundingBox inputBbox; diff --git a/src/aliceVision/panorama/laplacianPyramid.hpp b/src/aliceVision/panorama/laplacianPyramid.hpp index 1ad2956fa6..59d948d9b9 100644 --- a/src/aliceVision/panorama/laplacianPyramid.hpp +++ b/src/aliceVision/panorama/laplacianPyramid.hpp @@ -26,9 +26,9 @@ class LaplacianPyramid bool initialize(); - bool apply(const aliceVision::image::Image& source, - const aliceVision::image::Image& mask, - const aliceVision::image::Image& weights, + bool apply(aliceVision::image::Image& source, + aliceVision::image::Image& mask, + aliceVision::image::Image& weights, const BoundingBox &outputBoundingBox, const BoundingBox &contentBoudingBox); bool merge(const aliceVision::image::Image& oimg, diff --git a/src/software/pipeline/main_panoramaCompositing.cpp b/src/software/pipeline/main_panoramaCompositing.cpp index 8b4ae749ff..f95332a9ff 100644 --- a/src/software/pipeline/main_panoramaCompositing.cpp +++ b/src/software/pipeline/main_panoramaCompositing.cpp @@ -374,34 +374,33 @@ bool processImage(const PanoramaMap & panoramaMap, const std::string & composite continue; } - // Load image - const std::string imagePath = (fs::path(warpingFolder) / (std::to_string(viewCurrent) + ".exr")).string(); - ALICEVISION_LOG_TRACE("Load image with path " << imagePath); - image::Image source; - image::readImage(imagePath, source, image::EImageColorSpace::NO_CONVERSION); - - // Load mask - const std::string maskPath = (fs::path(warpingFolder) / (std::to_string(viewCurrent) + "_mask.exr")).string(); - ALICEVISION_LOG_TRACE("Load mask with path " << maskPath); - image::Image mask; - image::readImageDirect(maskPath, mask); - - // Load weights image if needed - image::Image weights; - if (needWeights) - { - const std::string weightsPath = (fs::path(warpingFolder) / (std::to_string(viewCurrent) + "_weight.exr")).string(); - ALICEVISION_LOG_TRACE("Load weights with path " << weightsPath); - image::readImage(weightsPath, weights, image::EImageColorSpace::NO_CONVERSION); - } - ALICEVISION_LOG_TRACE("Effective processing"); for (int indexIntersection = 0; indexIntersection < intersections.size(); indexIntersection++) { const BoundingBox & bbox = currentBoundingBoxes[indexIntersection]; const BoundingBox & bboxIntersect = intersections[indexIntersection]; + // Load image + const std::string imagePath = (fs::path(warpingFolder) / (std::to_string(viewCurrent) + ".exr")).string(); + ALICEVISION_LOG_TRACE("Load image with path " << imagePath); + image::Image source; + image::readImage(imagePath, source, image::EImageColorSpace::NO_CONVERSION); + // Load mask + const std::string maskPath = (fs::path(warpingFolder) / (std::to_string(viewCurrent) + "_mask.exr")).string(); + ALICEVISION_LOG_TRACE("Load mask with path " << maskPath); + image::Image mask; + image::readImageDirect(maskPath, mask); + + // Load weights image if needed + image::Image weights; + if (needWeights) + { + const std::string weightsPath = (fs::path(warpingFolder) / (std::to_string(viewCurrent) + "_weight.exr")).string(); + ALICEVISION_LOG_TRACE("Load weights with path " << weightsPath); + image::readImage(weightsPath, weights, image::EImageColorSpace::NO_CONVERSION); + } + if (needSeams) { int left = bboxIntersect.left - globalUnionBoundingBox.left; @@ -430,7 +429,10 @@ bool processImage(const PanoramaMap & panoramaMap, const std::string & composite image::Image submask(cutBoundingBox.width, cutBoundingBox.height); subsource = source.block(cutBoundingBox.top, cutBoundingBox.left, cutBoundingBox.height, cutBoundingBox.width); - submask = mask.block(cutBoundingBox.top, cutBoundingBox.left, cutBoundingBox.height, cutBoundingBox.width); + submask = mask.block(cutBoundingBox.top, cutBoundingBox.left, cutBoundingBox.height, cutBoundingBox.width); + + source = image::Image(); + mask = image::Image(); if (!compositer->append(subsource, submask, weights, referenceBoundingBox.left - panoramaBoundingBox.left + bboxIntersect.left - referenceBoundingBox.left , referenceBoundingBox.top - panoramaBoundingBox.top + bboxIntersect.top - referenceBoundingBox.top)) { From d53f81ac8a17df42e54aa96c4487b315496c6a0c Mon Sep 17 00:00:00 2001 From: Fabien Date: Wed, 20 Jan 2021 14:14:41 +0100 Subject: [PATCH 71/79] [panorama] enable multi threading in compositing --- src/aliceVision/panorama/panoramaMap.cpp | 2 +- src/aliceVision/panorama/panoramaMap.hpp | 2 +- .../pipeline/main_panoramaCompositing.cpp | 30 +++++++++++++++---- 3 files changed, 26 insertions(+), 8 deletions(-) diff --git a/src/aliceVision/panorama/panoramaMap.cpp b/src/aliceVision/panorama/panoramaMap.cpp index 24724d0231..cbd1f75ed5 100644 --- a/src/aliceVision/panorama/panoramaMap.cpp +++ b/src/aliceVision/panorama/panoramaMap.cpp @@ -52,7 +52,7 @@ bool PanoramaMap::intersect(const BoundingBox & box1, const BoundingBox & box2) return false; } -bool PanoramaMap::getOverlaps(std::list & overlaps, IndexT reference) const +bool PanoramaMap::getOverlaps(std::vector & overlaps, IndexT reference) const { if (_map.find(reference) == _map.end()) { diff --git a/src/aliceVision/panorama/panoramaMap.hpp b/src/aliceVision/panorama/panoramaMap.hpp index 2ff2685124..31f8ce3f54 100644 --- a/src/aliceVision/panorama/panoramaMap.hpp +++ b/src/aliceVision/panorama/panoramaMap.hpp @@ -24,7 +24,7 @@ class PanoramaMap bool append(IndexT index, const BoundingBox & box); - bool getOverlaps(std::list & overlaps, IndexT reference) const; + bool getOverlaps(std::vector & overlaps, IndexT reference) const; int getWidth() const { diff --git a/src/software/pipeline/main_panoramaCompositing.cpp b/src/software/pipeline/main_panoramaCompositing.cpp index f95332a9ff..3ef09e33d1 100644 --- a/src/software/pipeline/main_panoramaCompositing.cpp +++ b/src/software/pipeline/main_panoramaCompositing.cpp @@ -157,7 +157,7 @@ bool processImage(const PanoramaMap & panoramaMap, const std::string & composite // Get the list of input which should be processed for this reference view bounding box - std::list overlappingViews; + std::vector overlappingViews; if (!panoramaMap.getOverlaps(overlappingViews, viewReference)) { ALICEVISION_LOG_ERROR("Problem analyzing neighboorhood"); @@ -355,10 +355,17 @@ bool processImage(const PanoramaMap & panoramaMap, const std::string & composite return false; } - int posCurrent = 0; - for (IndexT viewCurrent : overlappingViews) + bool hasFailed = false; + + #pragma omp parallel for + for (int posCurrent = 0; posCurrent < overlappingViews.size(); posCurrent++) { - posCurrent++; + IndexT viewCurrent = overlappingViews[posCurrent]; + if (hasFailed) + { + continue; + } + ALICEVISION_LOG_INFO("Processing input " << posCurrent << "/" << overlappingViews.size()); // Compute list of intersection between this view and the reference view @@ -377,6 +384,11 @@ bool processImage(const PanoramaMap & panoramaMap, const std::string & composite ALICEVISION_LOG_TRACE("Effective processing"); for (int indexIntersection = 0; indexIntersection < intersections.size(); indexIntersection++) { + if (hasFailed) + { + continue; + } + const BoundingBox & bbox = currentBoundingBoxes[indexIntersection]; const BoundingBox & bboxIntersect = intersections[indexIntersection]; @@ -410,7 +422,7 @@ bool processImage(const PanoramaMap & panoramaMap, const std::string & composite if (!getMaskFromLabels(weights, referenceLabels, viewCurrent, left, top)) { ALICEVISION_LOG_ERROR("Error estimating seams image"); - return false; + hasFailed = true; } } @@ -437,11 +449,17 @@ bool processImage(const PanoramaMap & panoramaMap, const std::string & composite if (!compositer->append(subsource, submask, weights, referenceBoundingBox.left - panoramaBoundingBox.left + bboxIntersect.left - referenceBoundingBox.left , referenceBoundingBox.top - panoramaBoundingBox.top + bboxIntersect.top - referenceBoundingBox.top)) { ALICEVISION_LOG_INFO("Error in compositer append"); - return false; + hasFailed = true; + continue; } } } + if (hasFailed) + { + return false; + } + ALICEVISION_LOG_INFO("Terminate compositing for this view"); if (!compositer->terminate()) From 56013335886108c88d81fadac0434105f14b7533 Mon Sep 17 00:00:00 2001 From: Fabien Date: Wed, 20 Jan 2021 18:35:42 +0100 Subject: [PATCH 72/79] [panorama] prepare stuff for custom bouding box compositing --- src/aliceVision/panorama/panoramaMap.cpp | 20 +++++++---- src/aliceVision/panorama/panoramaMap.hpp | 4 +++ .../pipeline/main_panoramaCompositing.cpp | 34 ++++++++----------- 3 files changed, 31 insertions(+), 27 deletions(-) diff --git a/src/aliceVision/panorama/panoramaMap.cpp b/src/aliceVision/panorama/panoramaMap.cpp index cbd1f75ed5..b80e9a91a7 100644 --- a/src/aliceVision/panorama/panoramaMap.cpp +++ b/src/aliceVision/panorama/panoramaMap.cpp @@ -61,14 +61,14 @@ bool PanoramaMap::getOverlaps(std::vector & overlaps, IndexT reference) BoundingBox bbref = _map.at(reference); + return getOverlaps(overlaps, bbref); +} + +bool PanoramaMap::getOverlaps(std::vector & overlaps, const BoundingBox & referenceBoundingBox) const +{ for (auto it : _map) { - if (it.first == reference) - { - continue; - } - - if (intersect(bbref, it.second)) + if (intersect(referenceBoundingBox, it.second)) { overlaps.push_back(it.first); } @@ -77,12 +77,18 @@ bool PanoramaMap::getOverlaps(std::vector & overlaps, IndexT reference) return true; } + bool PanoramaMap::getIntersectionsList(std::vector & intersections, std::vector & currentBoundingBoxes, const IndexT & referenceIndex, const IndexT & otherIndex) const { BoundingBox referenceBoundingBox = _map.at(referenceIndex); + + return getIntersectionsList(intersections, currentBoundingBoxes, referenceBoundingBox, otherIndex); +} + +bool PanoramaMap::getIntersectionsList(std::vector & intersections, std::vector & currentBoundingBoxes, const BoundingBox & referenceBoundingBox, const IndexT & otherIndex) const +{ BoundingBox referenceBoundingBoxReduced = referenceBoundingBox.divide(_scale).dilate(_borderSize); - BoundingBox otherBoundingBox = _map.at(otherIndex); // Base compare diff --git a/src/aliceVision/panorama/panoramaMap.hpp b/src/aliceVision/panorama/panoramaMap.hpp index 31f8ce3f54..f52a2ed393 100644 --- a/src/aliceVision/panorama/panoramaMap.hpp +++ b/src/aliceVision/panorama/panoramaMap.hpp @@ -26,6 +26,8 @@ class PanoramaMap bool getOverlaps(std::vector & overlaps, IndexT reference) const; + bool getOverlaps(std::vector & overlaps, const BoundingBox & referenceBoundingBox) const; + int getWidth() const { return _panoramaWidth; @@ -59,6 +61,8 @@ class PanoramaMap } bool getIntersectionsList(std::vector & intersections, std::vector & currentBoundingBoxes, const IndexT & referenceIndex, const IndexT & otherIndex) const; + + bool getIntersectionsList(std::vector & intersections, std::vector & currentBoundingBoxes, const BoundingBox & referenceBoundingBox, const IndexT & otherIndex) const; bool optimizeChunks(std::vector> & chunks, int chunkSize); diff --git a/src/software/pipeline/main_panoramaCompositing.cpp b/src/software/pipeline/main_panoramaCompositing.cpp index 3ef09e33d1..614bc54307 100644 --- a/src/software/pipeline/main_panoramaCompositing.cpp +++ b/src/software/pipeline/main_panoramaCompositing.cpp @@ -114,16 +114,8 @@ std::unique_ptr buildMap(const sfmData::SfMData & sfmData, const st return ret; } -bool processImage(const PanoramaMap & panoramaMap, const std::string & compositerType, const std::string & warpingFolder, const std::string & labelsFilePath, const std::string & outputFolder, const image::EStorageDataType & storageDataType, const IndexT & viewReference, bool showBorders, bool showSeams) +bool processImage(const PanoramaMap & panoramaMap, const std::string & compositerType, const std::string & warpingFolder, const std::string & labelsFilePath, const std::string & outputFolder, const image::EStorageDataType & storageDataType, IndexT viewReference, const BoundingBox & referenceBoundingBox, bool showBorders, bool showSeams) { - // Get the input bounding box to define the ROI - BoundingBox referenceBoundingBox; - if (!panoramaMap.getBoundingBox(referenceBoundingBox, viewReference)) - { - ALICEVISION_LOG_ERROR("Problem getting reference bounding box"); - return false; - } - // The laplacian pyramid must also contains some pixels outside of the bounding box to make sure // there is a continuity between all the "views" of the panorama. BoundingBox panoramaBoundingBox = referenceBoundingBox; @@ -158,15 +150,11 @@ bool processImage(const PanoramaMap & panoramaMap, const std::string & composite // Get the list of input which should be processed for this reference view bounding box std::vector overlappingViews; - if (!panoramaMap.getOverlaps(overlappingViews, viewReference)) + if (!panoramaMap.getOverlaps(overlappingViews, referenceBoundingBox)) { ALICEVISION_LOG_ERROR("Problem analyzing neighboorhood"); return false; } - - // Add the reference input for simpler processing - overlappingViews.push_back(viewReference); - // Compute the bounding box of the intersections with the reference bounding box // (which may be larger than the reference Bounding box because of dilatation) @@ -176,7 +164,7 @@ bool processImage(const PanoramaMap & panoramaMap, const std::string & composite //Compute list of intersection between this view and the reference view std::vector intersections; std::vector currentBoundingBoxes; - if (!panoramaMap.getIntersectionsList(intersections, currentBoundingBoxes, viewReference, viewCurrent)) + if (!panoramaMap.getIntersectionsList(intersections, currentBoundingBoxes, referenceBoundingBox, viewCurrent)) { continue; } @@ -202,7 +190,7 @@ bool processImage(const PanoramaMap & panoramaMap, const std::string & composite // Compute list of intersection between this view and the reference view std::vector intersections; std::vector currentBoundingBoxes; - if (!panoramaMap.getIntersectionsList(intersections, currentBoundingBoxes, viewReference, viewCurrent)) + if (!panoramaMap.getIntersectionsList(intersections, currentBoundingBoxes, referenceBoundingBox, viewCurrent)) { continue; } @@ -371,7 +359,7 @@ bool processImage(const PanoramaMap & panoramaMap, const std::string & composite // Compute list of intersection between this view and the reference view std::vector intersections; std::vector currentBoundingBoxes; - if (!panoramaMap.getIntersectionsList(intersections, currentBoundingBoxes, viewReference, viewCurrent)) + if (!panoramaMap.getIntersectionsList(intersections, currentBoundingBoxes, referenceBoundingBox, viewCurrent)) { continue; } @@ -501,7 +489,7 @@ bool processImage(const PanoramaMap & panoramaMap, const std::string & composite // Compute list of intersection between this view and the reference view std::vector intersections; std::vector currentBoundingBoxes; - if (!panoramaMap.getIntersectionsList(intersections, currentBoundingBoxes, viewReference, viewCurrent)) + if (!panoramaMap.getIntersectionsList(intersections, currentBoundingBoxes, referenceBoundingBox, viewCurrent)) { continue; } @@ -718,9 +706,15 @@ int aliceVision_main(int argc, char** argv) ALICEVISION_LOG_INFO("processing input region " << posReference + 1 << "/" << chunk.size()); IndexT viewReference = chunk[posReference]; - if (viewReference == 0) continue; - if (!processImage(*panoramaMap, compositerType, warpingFolder, labelsFilepath, outputFolder, storageDataType, viewReference, showBorders, showSeams)) + BoundingBox referenceBoundingBox; + if (!panoramaMap->getBoundingBox(referenceBoundingBox, viewReference)) + { + ALICEVISION_LOG_ERROR("Invalid view ID as reference"); + return EXIT_FAILURE; + } + + if (!processImage(*panoramaMap, compositerType, warpingFolder, labelsFilepath, outputFolder, storageDataType, viewReference, referenceBoundingBox, showBorders, showSeams)) { succeeded = false; continue; From 45d990da1f3a03c4ecef632f986ddc11ee15fc46 Mon Sep 17 00:00:00 2001 From: Fabien Castan Date: Tue, 26 Jan 2021 19:05:02 +0100 Subject: [PATCH 73/79] [software] PanoramaCompositing: use omp_get_max_threads instead of omp_get_thread_limit not available on MSVC 2019 --- src/software/pipeline/main_panoramaCompositing.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/software/pipeline/main_panoramaCompositing.cpp b/src/software/pipeline/main_panoramaCompositing.cpp index 614bc54307..370bbed33b 100644 --- a/src/software/pipeline/main_panoramaCompositing.cpp +++ b/src/software/pipeline/main_panoramaCompositing.cpp @@ -698,7 +698,8 @@ int aliceVision_main(int argc, char** argv) bool succeeded = true; - omp_set_num_threads(std::min(omp_get_thread_limit(), std::max(0, maxThreads))); + if(maxThreads > 0) + omp_set_num_threads(std::min(omp_get_max_threads(), maxThreads)); //#pragma omp parallel for for (std::size_t posReference = 0; posReference < chunk.size(); posReference++) From 6204b1909bf6245ca067f8f43c99455cf96274cf Mon Sep 17 00:00:00 2001 From: Fabien Castan Date: Tue, 26 Jan 2021 19:05:42 +0100 Subject: [PATCH 74/79] [software] ImageProcessing!: remove useless code --- src/software/utils/main_imageProcessing.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/software/utils/main_imageProcessing.cpp b/src/software/utils/main_imageProcessing.cpp index 773fb07cac..34e56133f3 100644 --- a/src/software/utils/main_imageProcessing.cpp +++ b/src/software/utils/main_imageProcessing.cpp @@ -329,9 +329,6 @@ void processImage(image::Image& image, const ProcessingParams const oiio::ImageSpec imageSpecResized(nw, nh, nchannels, oiio::TypeDesc::FLOAT); const oiio::ImageSpec imageSpecOrigin(w, h, nchannels, oiio::TypeDesc::FLOAT); - oiio::ImageBuf bufferOrigin(imageSpecOrigin, image.data()); - oiio::ImageBuf bufferResized(imageSpecResized, rescaled.data()); - oiio::ImageBufAlgo::resample(bufferResized, bufferOrigin); const oiio::ImageBuf inBuf(imageSpecOrigin, image.data()); oiio::ImageBuf outBuf(imageSpecResized, rescaled.data()); From f308b230d22c0afb90fc541853e4effcb75df080 Mon Sep 17 00:00:00 2001 From: Fabien Castan Date: Tue, 26 Jan 2021 19:07:15 +0100 Subject: [PATCH 75/79] [software] PanoramaSeams: new max resolution option Allows to downscale input warps on-the-fly and compute seams in lower resolution. --- src/aliceVision/image/io.hpp | 45 +++++++++++++ src/aliceVision/image/pixelTypes.hpp | 35 ++++++++-- src/aliceVision/image/resampling.hpp | 67 ++++++++++++++++++++ src/aliceVision/panorama/boundingBox.cpp | 6 +- src/software/pipeline/main_panoramaSeams.cpp | 48 +++++++++----- 5 files changed, 178 insertions(+), 23 deletions(-) diff --git a/src/aliceVision/image/io.hpp b/src/aliceVision/image/io.hpp index 7906e239eb..055234a3e2 100644 --- a/src/aliceVision/image/io.hpp +++ b/src/aliceVision/image/io.hpp @@ -210,5 +210,50 @@ void writeImage(const std::string& path, const Image& image, EImageCo void writeImage(const std::string& path, const Image& image, EImageColorSpace imageColorSpace, const oiio::ParamValueList& metadata = oiio::ParamValueList()); void writeImage(const std::string& path, const Image& image, EImageColorSpace imageColorSpace, const oiio::ParamValueList& metadata = oiio::ParamValueList()); + +template +struct ColorTypeInfo +{ + // no size parameter, so no default value. + // An error will be raise at compile time if this type traits is not defined. +}; + +template <> +struct ColorTypeInfo +{ + static const int size = 1; + static const oiio::TypeDesc::BASETYPE typeDesc = oiio::TypeDesc::UINT8; +}; +template <> +struct ColorTypeInfo +{ + static const int size = 1; + static const oiio::TypeDesc::BASETYPE typeDesc = oiio::TypeDesc::FLOAT; +}; +template <> +struct ColorTypeInfo +{ + static const int size = 3; + static const oiio::TypeDesc::BASETYPE typeDesc = oiio::TypeDesc::UINT8; +}; +template <> +struct ColorTypeInfo +{ + static const int size = 3; + static const oiio::TypeDesc::BASETYPE typeDesc = oiio::TypeDesc::FLOAT; +}; +template <> +struct ColorTypeInfo +{ + static const int size = 4; + static const oiio::TypeDesc::BASETYPE typeDesc = oiio::TypeDesc::UINT8; +}; +template <> +struct ColorTypeInfo +{ + static const int size = 4; + static const oiio::TypeDesc::BASETYPE typeDesc = oiio::TypeDesc::FLOAT; +}; + } // namespace image } // namespace aliceVision diff --git a/src/aliceVision/image/pixelTypes.hpp b/src/aliceVision/image/pixelTypes.hpp index a4773d0efd..40e65ddc4f 100644 --- a/src/aliceVision/image/pixelTypes.hpp +++ b/src/aliceVision/image/pixelTypes.hpp @@ -420,11 +420,36 @@ namespace aliceVision // An error will be raise at compile time if this type traits is not defined. }; - template<> struct NbChannels{ int size = 1; }; - template<> struct NbChannels{ int size = 1; }; - template<> struct NbChannels{ int size = 3; }; - template<> struct NbChannels{ int size = 3; }; - template<> struct NbChannels{ int size = 4; }; + template <> + struct NbChannels + { + static const int size = 1; + }; + template <> + struct NbChannels + { + static const int size = 1; + }; + template <> + struct NbChannels + { + static const int size = 3; + }; + template <> + struct NbChannels + { + static const int size = 3; + }; + template <> + struct NbChannels + { + static const int size = 4; + }; + template <> + struct NbChannels + { + static const int size = 4; + }; } // namespace image diff --git a/src/aliceVision/image/resampling.hpp b/src/aliceVision/image/resampling.hpp index 2d29751499..1b673d88eb 100644 --- a/src/aliceVision/image/resampling.hpp +++ b/src/aliceVision/image/resampling.hpp @@ -8,6 +8,7 @@ #pragma once #include +#include namespace aliceVision { namespace image { @@ -37,6 +38,72 @@ namespace image { } } + template + void downscaleImage(const Image& src, Image& out, int downscale) + { + const int new_width = src.Width() / downscale; + const int new_height = src.Height() / downscale; + + out.resize(new_width, new_height); + + const Sampler2d sampler; + const float downscalef = downscale; + for(int i = 0; i < new_height; ++i) + { + for(int j = 0; j < new_width; ++j) + { + // Use .5f offset to ensure mid pixel and correct sampling + out(i, j) = sampler(src, downscalef * (i + .5f), downscalef * (j + .5f)); + } + } + } + + template + void downscaleImageInplace(ImageType& inout, int downscale) + { + if(downscale <= 1) + return; + ALICEVISION_LOG_TRACE("downscaleImageInplace in: " << inout.Width() << "x" << inout.Height()); + /* + { + // Gaussian filter + Nearest Neighbor: could be expensive for very large resolutions with large downscale factor. + Image otherImg; + const double sigma = downscale * 0.5; + ImageGaussianFilter(inout, sigma, otherImg); // TODO: error on RGBColor + downscaleImage(otherImg, inout, downscale); + ALICEVISION_LOG_INFO("downscaleImageInplace otherImg: " << otherImg.Width() << "x" << otherImg.Height()); + } + { + // Simple bilinear interpolation: only valid for <= 2x downscale + ImageType otherImg; + downscaleImage(inout, otherImg, downscale); + std::swap(inout, otherImg); + }*/ + { + // Rely on OpenImageIO to do the downscaling + const unsigned int w = inout.Width(); + const unsigned int h = inout.Height(); + const unsigned int nchannels = inout.Channels(); + + const unsigned int nw = (unsigned int)(floor(float(w) / downscale)); + const unsigned int nh = (unsigned int)(floor(float(h) / downscale)); + + ImageType rescaled(nw, nh); + + const oiio::ImageSpec imageSpecOrigin(w, h, nchannels, ColorTypeInfo::typeDesc); + const oiio::ImageSpec imageSpecResized(nw, nh, nchannels, ColorTypeInfo::typeDesc); + + const oiio::ImageBuf inBuf(imageSpecOrigin, inout.data()); + oiio::ImageBuf outBuf(imageSpecResized, rescaled.data()); + + oiio::ImageBufAlgo::resize(outBuf, inBuf); + + inout.swap(rescaled); + } + ALICEVISION_LOG_TRACE("downscaleImageInplace out: " << inout.Width() << "x" << inout.Height()); + } + + /** ** @brief Ressample an image using given sampling positions ** @param src Input image diff --git a/src/aliceVision/panorama/boundingBox.cpp b/src/aliceVision/panorama/boundingBox.cpp index a624f6484b..86fd830e60 100644 --- a/src/aliceVision/panorama/boundingBox.cpp +++ b/src/aliceVision/panorama/boundingBox.cpp @@ -5,11 +5,11 @@ namespace aliceVision std::ostream& operator<<(std::ostream& os, const BoundingBox& in) { - os << in.left << " "; + os << in.left << ","; os << in.top << " "; - os << in.width << " "; + os << in.width << "x"; os << in.height; - + return os; } diff --git a/src/software/pipeline/main_panoramaSeams.cpp b/src/software/pipeline/main_panoramaSeams.cpp index b1f297c06f..fae3f2d020 100644 --- a/src/software/pipeline/main_panoramaSeams.cpp +++ b/src/software/pipeline/main_panoramaSeams.cpp @@ -42,11 +42,11 @@ namespace po = boost::program_options; namespace bpt = boost::property_tree; namespace fs = boost::filesystem; -bool computeWTALabels(image::Image & labels, const std::vector> & views, const std::string & inputPath, std::pair & panoramaSize) +bool computeWTALabels(image::Image & labels, const std::vector> & views, const std::string & inputPath, const std::pair & panoramaSize, int downscale) { ALICEVISION_LOG_INFO("Estimating initial labels for panorama"); - WTASeams seams(panoramaSize.first, panoramaSize.second); + WTASeams seams(panoramaSize.first / downscale, panoramaSize.second / downscale); for (const auto& viewIt : views) { @@ -57,17 +57,19 @@ bool computeWTALabels(image::Image & labels, const std::vector mask; image::readImageDirect(maskPath, mask); + image::downscaleImageInplace(mask, downscale); // Get offset oiio::ParamValueList metadata = image::readImageMetadata(maskPath); - const std::size_t offsetX = metadata.find("AliceVision:offsetX")->get_int(); - const std::size_t offsetY = metadata.find("AliceVision:offsetY")->get_int(); + const std::size_t offsetX = metadata.find("AliceVision:offsetX")->get_int() / downscale; + const std::size_t offsetY = metadata.find("AliceVision:offsetY")->get_int() / downscale; // Load Weights const std::string weightsPath = (fs::path(inputPath) / (std::to_string(viewId) + "_weight.exr")).string(); ALICEVISION_LOG_TRACE("Load weights with path " << weightsPath); image::Image weights; image::readImage(weightsPath, weights, image::EImageColorSpace::NO_CONVERSION); + image::downscaleImageInplace(weights, downscale); if (!seams.appendWithLoop(mask, weights, viewId, offsetX, offsetY)) { @@ -80,14 +82,16 @@ bool computeWTALabels(image::Image & labels, const std::vector & labels, const std::vector> & views, const std::string & inputPath, std::pair & panoramaSize, int smallestViewScale) -{ +bool computeGCLabels(image::Image& labels, const std::vector>& views, + const std::string& inputPath, std::pair& panoramaSize, int smallestViewScale, + int downscale) +{ ALICEVISION_LOG_INFO("Estimating smart seams for panorama"); int pyramidSize = 1 + std::max(0, smallestViewScale - 1); ALICEVISION_LOG_INFO("Graphcut pyramid size is " << pyramidSize); - HierarchicalGraphcutSeams seams(panoramaSize.first, panoramaSize.second, pyramidSize); + HierarchicalGraphcutSeams seams(panoramaSize.first / downscale, panoramaSize.second / downscale, pyramidSize); if (!seams.initialize(labels)) { @@ -103,21 +107,22 @@ bool computeGCLabels(image::Image & labels, const std::vector mask; image::readImageDirect(maskPath, mask); + image::downscaleImageInplace(mask, downscale); // Load Color const std::string colorsPath = (fs::path(inputPath) / (std::to_string(viewId) + ".exr")).string(); ALICEVISION_LOG_TRACE("Load colors with path " << colorsPath); image::Image colors; image::readImage(colorsPath, colors, image::EImageColorSpace::NO_CONVERSION); + image::downscaleImageInplace(colors, downscale); // Get offset oiio::ParamValueList metadata = image::readImageMetadata(maskPath); - const std::size_t offsetX = metadata.find("AliceVision:offsetX")->get_int(); - const std::size_t offsetY = metadata.find("AliceVision:offsetY")->get_int(); - + const std::size_t offsetX = metadata.find("AliceVision:offsetX")->get_int() / downscale; + const std::size_t offsetY = metadata.find("AliceVision:offsetY")->get_int() / downscale; // Append to graph cut - if (!seams.append(colors, mask, viewId, offsetX, offsetY)) + if (!seams.append(colors, mask, viewId, offsetX, offsetY)) { return false; } @@ -160,6 +165,7 @@ int aliceVision_main(int argc, char** argv) std::string outputLabels; std::string temporaryCachePath; + int maxPanoramaWidth = 3000; bool useGraphCut = true; image::EStorageDataType storageDataType = image::EStorageDataType::Float; @@ -181,7 +187,8 @@ int aliceVision_main(int argc, char** argv) // Description of optional parameters po::options_description optionalParams("Optional parameters"); optionalParams.add_options() - ("useGraphCut,g", po::value(&useGraphCut)->default_value(useGraphCut), "Do we use graphcut for ghost removal ?"); + ("maxWidth", po::value(&maxPanoramaWidth)->required(), "Max Panorama Width.") + ("useGraphCut,g", po::value(&useGraphCut)->default_value(useGraphCut), "Enable graphcut algorithm to improve seams."); allParams.add(optionalParams); // Setup log level given command line @@ -234,6 +241,7 @@ int aliceVision_main(int argc, char** argv) int tileSize; std::pair panoramaSize; + int downscaleFactor = 1; { const IndexT viewId = *sfmData.getValidViews().begin(); const std::string viewFilepath = (fs::path(warpingFolder) / (std::to_string(viewId) + ".exr")).string(); @@ -256,7 +264,15 @@ int aliceVision_main(int argc, char** argv) return EXIT_FAILURE; } - ALICEVISION_LOG_INFO("Output labels size set to " << panoramaSize.first << "x" << panoramaSize.second); + if(maxPanoramaWidth > 0 && panoramaSize.first > maxPanoramaWidth) + { + downscaleFactor = std::ceil(panoramaSize.first / double(maxPanoramaWidth)); + } + + ALICEVISION_LOG_INFO("Input panorama size is " << panoramaSize.first << "x" << panoramaSize.second); + ALICEVISION_LOG_INFO("Downscale factor set to " << downscaleFactor); + ALICEVISION_LOG_INFO("Output labels size set to " << (panoramaSize.first / downscaleFactor) << "x" + << (panoramaSize.second / downscaleFactor)); } //Get a list of views ordered by their image scale @@ -277,6 +293,8 @@ int aliceVision_main(int argc, char** argv) const std::string maskPath = (fs::path(warpingFolder) / (std::to_string(viewId) + "_mask.exr")).string(); int width, height; image::readImageMetadata(maskPath, width, height); + width /= downscaleFactor; + height /= downscaleFactor; //Estimate scale int scale = getGraphcutOptimalScale(width, height); @@ -288,7 +306,7 @@ int aliceVision_main(int argc, char** argv) ALICEVISION_LOG_INFO(views.size() << " views to process"); image::Image labels; - if (!computeWTALabels(labels, views, warpingFolder, panoramaSize)) + if(!computeWTALabels(labels, views, warpingFolder, panoramaSize, downscaleFactor)) { ALICEVISION_LOG_ERROR("Error computing initial labels"); return EXIT_FAILURE; @@ -296,7 +314,7 @@ int aliceVision_main(int argc, char** argv) if (useGraphCut) { - if (!computeGCLabels(labels, views, warpingFolder, panoramaSize, smallestScale)) + if(!computeGCLabels(labels, views, warpingFolder, panoramaSize, smallestScale, downscaleFactor)) { ALICEVISION_LOG_ERROR("Error computing graph cut labels"); return EXIT_FAILURE; From b1a9a9ba1ffb540bfd9be1fc159e56f9918a82b8 Mon Sep 17 00:00:00 2001 From: Fabien Castan Date: Tue, 26 Jan 2021 19:24:55 +0100 Subject: [PATCH 76/79] [image] linux build fixes --- src/aliceVision/image/resampling.hpp | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/aliceVision/image/resampling.hpp b/src/aliceVision/image/resampling.hpp index 1b673d88eb..463c2bc455 100644 --- a/src/aliceVision/image/resampling.hpp +++ b/src/aliceVision/image/resampling.hpp @@ -7,14 +7,22 @@ #pragma once +#include "io.hpp" + #include #include +#include +#include +#include + +namespace oiio = OIIO; + namespace aliceVision { namespace image { /** - ** Half sample an image (ie reduce it's size by a factor 2) using bilinear interpolation + ** Half sample an image (ie reduce its size by a factor 2) using bilinear interpolation ** @param src input image ** @param out output image **/ @@ -90,8 +98,8 @@ namespace image { ImageType rescaled(nw, nh); - const oiio::ImageSpec imageSpecOrigin(w, h, nchannels, ColorTypeInfo::typeDesc); - const oiio::ImageSpec imageSpecResized(nw, nh, nchannels, ColorTypeInfo::typeDesc); + const oiio::ImageSpec imageSpecOrigin(w, h, nchannels, ColorTypeInfo::typeDesc); + const oiio::ImageSpec imageSpecResized(nw, nh, nchannels, ColorTypeInfo::typeDesc); const oiio::ImageBuf inBuf(imageSpecOrigin, inout.data()); oiio::ImageBuf outBuf(imageSpecResized, rescaled.data()); From d08ac36944af546ee74363a258311615546b1363 Mon Sep 17 00:00:00 2001 From: Fabien Castan Date: Wed, 27 Jan 2021 14:09:23 +0100 Subject: [PATCH 77/79] [software] PanoramaCompositing: fixes for non-reconstructed views --- src/software/pipeline/main_panoramaCompositing.cpp | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/software/pipeline/main_panoramaCompositing.cpp b/src/software/pipeline/main_panoramaCompositing.cpp index 370bbed33b..2ce76453d1 100644 --- a/src/software/pipeline/main_panoramaCompositing.cpp +++ b/src/software/pipeline/main_panoramaCompositing.cpp @@ -77,6 +77,9 @@ std::unique_ptr buildMap(const sfmData::SfMData & sfmData, const st for (const auto& viewIt : sfmData.getViews()) { + if(!sfmData.isPoseAndIntrinsicDefined(viewIt.first)) + continue; + // Load mask const std::string maskPath = (fs::path(inputPath) / (std::to_string(viewIt.first) + "_mask.exr")).string(); ALICEVISION_LOG_TRACE("Load metadata of mask with path " << maskPath); @@ -694,6 +697,12 @@ int aliceVision_main(int argc, char** argv) return EXIT_FAILURE; } + if(rangeIteration >= chunks.size()) + { + // nothing to compute for this chunk + return EXIT_SUCCESS; + } + const std::vector & chunk = chunks[rangeIteration]; bool succeeded = true; @@ -707,6 +716,8 @@ int aliceVision_main(int argc, char** argv) ALICEVISION_LOG_INFO("processing input region " << posReference + 1 << "/" << chunk.size()); IndexT viewReference = chunk[posReference]; + if(!sfmData.isPoseAndIntrinsicDefined(viewReference)) + continue; BoundingBox referenceBoundingBox; if (!panoramaMap->getBoundingBox(referenceBoundingBox, viewReference)) From e572602b46f7ced6528b25feb4aade6391ed9c8a Mon Sep 17 00:00:00 2001 From: Fabien Castan Date: Wed, 27 Jan 2021 14:09:51 +0100 Subject: [PATCH 78/79] [software] LdrToHdr: disable highlight by default --- src/software/pipeline/main_LdrToHdrMerge.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/software/pipeline/main_LdrToHdrMerge.cpp b/src/software/pipeline/main_LdrToHdrMerge.cpp index 4b9ccac172..9b7539a02e 100644 --- a/src/software/pipeline/main_LdrToHdrMerge.cpp +++ b/src/software/pipeline/main_LdrToHdrMerge.cpp @@ -57,7 +57,7 @@ int aliceVision_main(int argc, char** argv) int offsetRefBracketIndex = 0; hdr::EFunctionType fusionWeightFunction = hdr::EFunctionType::GAUSSIAN; - float highlightCorrectionFactor = 1.0f; + float highlightCorrectionFactor = 0.0f; float highlightTargetLux = 120000.0f; image::EStorageDataType storageDataType = image::EStorageDataType::Float; From 0d7062b5fc7770b65b5cd25306f3652b9285f2f1 Mon Sep 17 00:00:00 2001 From: Fabien Date: Mon, 1 Feb 2021 10:31:13 +0100 Subject: [PATCH 79/79] [panorama] error in graphcut distance acceleration criteria --- src/aliceVision/panorama/graphcut.hpp | 4 ++-- src/aliceVision/panorama/seams.cpp | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/aliceVision/panorama/graphcut.hpp b/src/aliceVision/panorama/graphcut.hpp index 2a7fbb524d..ec5d0d6e39 100644 --- a/src/aliceVision/panorama/graphcut.hpp +++ b/src/aliceVision/panorama/graphcut.hpp @@ -533,7 +533,7 @@ class GraphcutSeams float d2 = float(distanceMap(i, j)); float d = sqrt(d2); - if (d2 > _maximal_distance_change + 1.0f) + if (d > _maximal_distance_change + 1.0f) { graphCutInput(i, j).clear(); } @@ -553,7 +553,7 @@ class GraphcutSeams BoundingBox inputBb = localBbox; inputBb.left = 0; inputBb.top = 0; - + if (!loopyImageAssign(_labels, localLabels, localBbox, inputBb)) { return false; diff --git a/src/aliceVision/panorama/seams.cpp b/src/aliceVision/panorama/seams.cpp index 838320470c..cbb9d1a59b 100644 --- a/src/aliceVision/panorama/seams.cpp +++ b/src/aliceVision/panorama/seams.cpp @@ -427,11 +427,13 @@ bool HierarchicalGraphcutSeams::process() if (level == _countLevels - 1) { - _graphcuts[level].setMaximalDistance(w + h); + _graphcuts[level].setMaximalDistance(sqrt(w*w + h*h)); } else { - _graphcuts[level].setMaximalDistance(200); + double sw = double(0.2 * w); + double sh = double(0.2 * h); + _graphcuts[level].setMaximalDistance(sqrt(sw*sw + sh*sh)); }